<commit_before>#!/usr/bin/python

import dbus
import dbus.service
import gobject

from dbus.mainloop.glib import DBusGMainLoop
DBusGMainLoop(set_as_default=True)

AVAHI_NAME = 'org.freedesktop.Avahi'
AVAHI_IFACE_SERVER = 'org.freedesktop.Avahi.Server'

class Avahi(dbus.service.Object):
    def __init__(self):
        bus = dbus.SystemBus()
        name = dbus.service.BusName(AVAHI_NAME, bus)
        dbus.service.Object.__init__(self, conn=bus, object_path='/',
                                     bus_name=name)

        self._entry_groups = []

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='s')
    def GetVersionString(self):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='u')
    def GetAPIVersion(self):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='s')
    def GetHostName(self):
        return 'avahimock_hostname'

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='s', out_signature='')
    def SetHostName(self, name):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='s')
    def GetHostNameFqdn(self):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='s')
    def GetDomainName(self):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='b')
    def IsNSSSupportAvailable(self):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='i')
    def GetState(self):
        raise NotImplementedError()

    @dbus.service.signal(dbus_interface=AVAHI_IFACE_SERVER, signature='is')
    def StateChanged(self, state, error):
        pass

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='u')
    def GetLocalServiceCookie(self):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='s', out_signature='s')
    def GetAlternativeHostName(self, name):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='s', out_signature='s')
    def GetAlternativeServiceName(self, name):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='i', out_signature='s')
    def GetNetworkInterfaceNameByIndex(self, index):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='s', out_signature='i')
    def GetNetworkInterfaceIndexByName(self, name):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisiu', out_signature='iisisu')
    def ResolveHostName(self, interface, protocol, name, aprotocol, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisu', out_signature='iiissu')
    def ResolveAddress(self, interface, protocol, address, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisssiu', out_signature='iissssisqaayu')
    def ResolveService(self, interface, protocol, name, type_, domain, aprotocol,
                       flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='o')
    def EntryGroupNew(self):
        entry_group = EntryGroup()
        self._entry_groups.append(entry_group)
        return entry_group.object_path

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisiu', out_signature='o')
    def DomainBrowserNew(self, interface, protocol, domain, btype, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisu', out_signature='o')
    def ServiceTypeBrowserNew(self, interface, protocol, domain, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iissu', out_signature='o')
    def ServiceBrowserNew(self, interface, protocol, type_, domain, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisssiu', out_signature='o')
    def ServiceResolverNew(self, interface, protocol, name, type_, domain, aprotocol, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisiu', out_signature='o')
    def HostNameResolverNew(self, interface, protocol, name, aprotocol, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisu', out_signature='o')
    def AddressResolverNew(self, interface, protocol, address, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisqqu', out_signature='o')
    def RecordBrowserNew(self, interface, protocol, name, clazz, type_, flags):
        raise NotImplementedError()

class EntryGroup(dbus.service.Object):
    def __init__(self):
        bus = dbus.SystemBus()
        self.object_path = '/Client%u/EntryGroup%u' % (1, 1)
        dbus.service.Object.__init__(self, conn=bus,
                                     object_path=self.object_path)

avahi = Avahi()

loop = gobject.MainLoop()
loop.run()
<commit_msg>Implement o.f.Avahi.Server.GetHostNameFqdn and GetDomainName<commit_after>#!/usr/bin/python

import dbus
import dbus.service
import gobject

from dbus.mainloop.glib import DBusGMainLoop
DBusGMainLoop(set_as_default=True)

AVAHI_NAME = 'org.freedesktop.Avahi'
AVAHI_IFACE_SERVER = 'org.freedesktop.Avahi.Server'

class Avahi(dbus.service.Object):
    def __init__(self):
        bus = dbus.SystemBus()
        name = dbus.service.BusName(AVAHI_NAME, bus)
        dbus.service.Object.__init__(self, conn=bus, object_path='/',
                                     bus_name=name)

        self._entry_groups = []

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='s')
    def GetVersionString(self):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='u')
    def GetAPIVersion(self):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='s')
    def GetHostName(self):
        return 'avahimock_hostname'

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='s', out_signature='')
    def SetHostName(self, name):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='s')
    def GetHostNameFqdn(self):
        return 'avahimock_hostname.local'

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='s')
    def GetDomainName(self):
        return 'local'

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='b')
    def IsNSSSupportAvailable(self):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='i')
    def GetState(self):
        raise NotImplementedError()

    @dbus.service.signal(dbus_interface=AVAHI_IFACE_SERVER, signature='is')
    def StateChanged(self, state, error):
        pass

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='u')
    def GetLocalServiceCookie(self):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='s', out_signature='s')
    def GetAlternativeHostName(self, name):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='s', out_signature='s')
    def GetAlternativeServiceName(self, name):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='i', out_signature='s')
    def GetNetworkInterfaceNameByIndex(self, index):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='s', out_signature='i')
    def GetNetworkInterfaceIndexByName(self, name):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisiu', out_signature='iisisu')
    def ResolveHostName(self, interface, protocol, name, aprotocol, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisu', out_signature='iiissu')
    def ResolveAddress(self, interface, protocol, address, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisssiu', out_signature='iissssisqaayu')
    def ResolveService(self, interface, protocol, name, type_, domain, aprotocol,
                       flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='', out_signature='o')
    def EntryGroupNew(self):
        entry_group = EntryGroup()
        self._entry_groups.append(entry_group)
        return entry_group.object_path

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisiu', out_signature='o')
    def DomainBrowserNew(self, interface, protocol, domain, btype, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisu', out_signature='o')
    def ServiceTypeBrowserNew(self, interface, protocol, domain, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iissu', out_signature='o')
    def ServiceBrowserNew(self, interface, protocol, type_, domain, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisssiu', out_signature='o')
    def ServiceResolverNew(self, interface, protocol, name, type_, domain, aprotocol, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisiu', out_signature='o')
    def HostNameResolverNew(self, interface, protocol, name, aprotocol, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisu', out_signature='o')
    def AddressResolverNew(self, interface, protocol, address, flags):
        raise NotImplementedError()

    @dbus.service.method(dbus_interface=AVAHI_IFACE_SERVER,
                         in_signature='iisqqu', out_signature='o')
    def RecordBrowserNew(self, interface, protocol, name, clazz, type_, flags):
        raise NotImplementedError()

class EntryGroup(dbus.service.Object):
    def __init__(self):
        bus = dbus.SystemBus()
        self.object_path = '/Client%u/EntryGroup%u' % (1, 1)
        dbus.service.Object.__init__(self, conn=bus,
                                     object_path=self.object_path)

avahi = Avahi()

loop = gobject.MainLoop()
loop.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys, os, re
import json
import datetime, iso8601

# Save to Mongo
from bson import json_util
import pymongo_spark
pymongo_spark.activate()

# Pass date and base path to main() from airflow
def main(iso_date, base_path):
  APP_NAME = "fetch_prediction_requests.py"

  # If there is no SparkSession, create the environment
  try:
    sc and spark
  except NameError as e:
    import findspark
    findspark.init()
    import pyspark
    import pyspark.sql

    sc = pyspark.SparkContext()
    spark = pyspark.sql.SparkSession(sc).builder.appName(APP_NAME).getOrCreate()

  # Get today and tomorrow's dates as iso strings to scope query
  today_dt = iso8601.parse_date(iso_date)
  rounded_today = today_dt.date()
  iso_today = rounded_today.isoformat()
  rounded_tomorrow_dt = rounded_today + datetime.timedelta(days=1)
  iso_tomorrow = rounded_tomorrow_dt.isoformat()

  # Create mongo query string for today's data
  mongo_query_string = """{{
    "Timestamp": {{
      "$gte": "{iso_today}",
      "$lte": "{iso_tomorrow}"
    }}
  }}""".format(
    iso_today=iso_today,
    iso_tomorrow=iso_tomorrow
  )
  mongo_query_string = mongo_query_string.replace('\n', '')

  # Create the config object with the query string
  mongo_query_config = dict()
  mongo_query_config["mongo.input.query"] = mongo_query_string

  # Load the day's requests using pymongo_spark
  prediction_requests = sc.mongoRDD(
    'mongodb://localhost:27017/agile_data_science.prediction_tasks',
    config=mongo_query_config
  )

  # Build the day's output path: a date based primary key directory structure
  today_output_path = "{}/data/prediction_tasks.json/{}".format(
    base_path,
    iso_today
  )

  # Generate json records
  prediction_requests_json = prediction_requests.map(json_util.dumps)

  # Write/replace today's output path
  os.system("rm -rf {}".format(today_output_path))
  prediction_requests_json.saveAsTextFile(today_output_path)

if __name__ == "__main__":
  main(sys.argv[1], sys.argv[2])
<commit_msg>Cleanup formatting one line<commit_after>#!/usr/bin/env python

import sys, os, re
import json
import datetime, iso8601

# Save to Mongo
from bson import json_util
import pymongo_spark
pymongo_spark.activate()

# Pass date and base path to main() from airflow
def main(iso_date, base_path):

  APP_NAME = "fetch_prediction_requests.py"

  # If there is no SparkSession, create the environment
  try:
    sc and spark
  except NameError as e:
    import findspark
    findspark.init()
    import pyspark
    import pyspark.sql

    sc = pyspark.SparkContext()
    spark = pyspark.sql.SparkSession(sc).builder.appName(APP_NAME).getOrCreate()

  # Get today and tomorrow's dates as iso strings to scope query
  today_dt = iso8601.parse_date(iso_date)
  rounded_today = today_dt.date()
  iso_today = rounded_today.isoformat()
  rounded_tomorrow_dt = rounded_today + datetime.timedelta(days=1)
  iso_tomorrow = rounded_tomorrow_dt.isoformat()

  # Create mongo query string for today's data
  mongo_query_string = """{{
    "Timestamp": {{
      "$gte": "{iso_today}",
      "$lte": "{iso_tomorrow}"
    }}
  }}""".format(
    iso_today=iso_today,
    iso_tomorrow=iso_tomorrow
  )
  mongo_query_string = mongo_query_string.replace('\n', '')

  # Create the config object with the query string
  mongo_query_config = dict()
  mongo_query_config["mongo.input.query"] = mongo_query_string

  # Load the day's requests using pymongo_spark
  prediction_requests = sc.mongoRDD(
    'mongodb://localhost:27017/agile_data_science.prediction_tasks',
    config=mongo_query_config
  )

  # Build the day's output path: a date based primary key directory structure
  today_output_path = "{}/data/prediction_tasks.json/{}".format(
    base_path,
    iso_today
  )

  # Generate json records
  prediction_requests_json = prediction_requests.map(json_util.dumps)

  # Write/replace today's output path
  os.system("rm -rf {}".format(today_output_path))
  prediction_requests_json.saveAsTextFile(today_output_path)

if __name__ == "__main__":
  main(sys.argv[1], sys.argv[2])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#

import urllib
import os
import sys
import pickle
import datetime
import optparse
import logging

from cik import Session, Base, init_model, cikUIK
from sqlalchemy import create_engine

logging.basicConfig(level = logging.INFO)

all_regions = [ 'adygei',
    "altai_rep", "bashkortostan", "buriat", "dagestan", "ingush",
    "kabardin-balkar", "kalmyk", "karachaev-cherkess", "karel", "komi",
    "mari-el", "mordov", "yakut", "n_osset-alania", "tatarstan",
    "tyva", "udmurt", "khakas", "chechen", "chuvash",
    "altai_terr", "zabkray", "kamchatka_krai", "krasnodar", "krasnoyarsk",
    "permkrai", "primorsk", "stavropol", "khabarovsk", "amur",
    "arkhangelsk", "astrakhan", "belgorod", "bryansk", "vladimir",
    "volgograd", "vologod", "voronezh", "ivanovo", "irkutsk",
    "kaliningrad", "kaluga", "kemerovo", "kirov", "kostroma",
    "kurgan", "kursk", "leningrad-reg", "lipetsk", "magadan",
    "moscow_reg", "murmansk", "nnov", "novgorod", "novosibirsk",
    "omsk", "orenburg", "orel", "penza", "pskov",
    "rostov", "ryazan", "samara", "saratov", "sakhalin",
    "sverdlovsk", "smolensk", "tambov", "tver", "tomsk",
    "tula", "tyumen", "ulyanovsk", "chelyabinsk", "yaroslavl",
    "moscow_city", "st-petersburg", "jewish_aut", "nenetsk", "khantu-mansy",
    "chukot", "yamal-nenetsk",
]

if __name__ == '__main__':
    parser = optparse.OptionParser("usage: %prog")
    parser.add_option("-r", "--region", dest="regions",
                      default='all', help=u'какой регион выкачать')
    parser.add_option("--db", dest="path_db",
                      default='cik.sqlite', help=u'путь к дб')

    (options, args) = parser.parse_args()

    engine = create_engine('sqlite:///%s' % (options.path_db, ))
    init_model(engine)

    if options.regions == 'all':
        down_regions = all_regions
    else:
        down_regions = options.regions.split(',')

    for reg in down_regions:
        if reg not in all_regions: continue

        ik = Session.query(cikUIK).filter( cikUIK.region == reg, cikUIK.type_ik == 'ik' ).first()
        if not ik:
            ik = cikUIK( region = reg, type_ik = 'ik', url='http://www.%s.vybory.izbirkom.ru/ik/' % (reg, ) )
            ik.iz_id = -1
            Session.add(ik)

        vals = ik.parse(recursion=True)

<commit_msg>down.py -vv - уровень дебага<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#

import urllib
import os
import sys
import pickle
import datetime
import optparse
import logging

all_regions = [ 'adygei',
    "altai_rep", "bashkortostan", "buriat", "dagestan", "ingush",
    "kabardin-balkar", "kalmyk", "karachaev-cherkess", "karel", "komi",
    "mari-el", "mordov", "yakut", "n_osset-alania", "tatarstan",
    "tyva", "udmurt", "khakas", "chechen", "chuvash",
    "altai_terr", "zabkray", "kamchatka_krai", "krasnodar", "krasnoyarsk",
    "permkrai", "primorsk", "stavropol", "khabarovsk", "amur",
    "arkhangelsk", "astrakhan", "belgorod", "bryansk", "vladimir",
    "volgograd", "vologod", "voronezh", "ivanovo", "irkutsk",
    "kaliningrad", "kaluga", "kemerovo", "kirov", "kostroma",
    "kurgan", "kursk", "leningrad-reg", "lipetsk", "magadan",
    "moscow_reg", "murmansk", "nnov", "novgorod", "novosibirsk",
    "omsk", "orenburg", "orel", "penza", "pskov",
    "rostov", "ryazan", "samara", "saratov", "sakhalin",
    "sverdlovsk", "smolensk", "tambov", "tver", "tomsk",
    "tula", "tyumen", "ulyanovsk", "chelyabinsk", "yaroslavl",
    "moscow_city", "st-petersburg", "jewish_aut", "nenetsk", "khantu-mansy",
    "chukot", "yamal-nenetsk",
]

if __name__ == '__main__':
    parser = optparse.OptionParser("usage: %prog")
    parser.add_option("-r", "--region", dest="regions",
                      default='all', help=u'какой регион выкачать (через ,)')
    parser.add_option("--db", dest="path_db",
                      default='cik.sqlite', help=u'путь к дб')
    parser.add_option("-v", "--verbose", dest="debug", action="count",
                      default=False, help=u'verbose level')

    (options, args) = parser.parse_args()

    if options.debug == 1: logging.basicConfig(level = logging.INFO)
    elif options.debug >= 2: logging.basicConfig(level = logging.DEBUG)
    else: logging.basicConfig(level = logging.ERROR)

    from cik import Session, Base, init_model, cikUIK
    from sqlalchemy import create_engine

    engine = create_engine('sqlite:///%s' % (options.path_db, ))
    init_model(engine)

    if options.regions == 'all':
        down_regions = all_regions
    else:
        down_regions = options.regions.split(',')

    for reg in down_regions:
        if reg not in all_regions: continue

        ik = Session.query(cikUIK).filter( cikUIK.region == reg, cikUIK.type_ik == 'ik' ).first()
        if not ik:
            ik = cikUIK( region = reg, type_ik = 'ik', url='http://www.%s.vybory.izbirkom.ru/ik/' % (reg, ) )
            ik.iz_id = -1
            Session.add(ik)

        vals = ik.parse(recursion=True)
        Session.commit()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os, sys

if (len(sys.argv) == 1):
    print("Usage: 'python3 deploy.py <server_path>'")
else:
    path = sys.argv[1] + "src/main/webapp"
    filename = "tmp.tar.gz"
    os.system("tar -zcvf %s tmp/" % filename)
    os.system("mv tmp.tar.gz %s" % path)
    os.system("tar -zxvf %s/%s --directory=%s --strip-components 1" % (path, filename, path))
    os.system("rm %s/%s" % (path, filename))
    os.system("mv %s/index.html %s/WEB-INF/templates/tickets.html" % (path, path))
<commit_msg>Update deploy scripts<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os, sys

if (len(sys.argv) == 1):
    print("Usage: 'python3 deploy.py <server_path>'")
else:
    path = sys.argv[1] + "src/main/webapp"
    filename = "tmp.tar.gz"
    os.system("gulp clean")
    os.system("gulp build")
    os.system("tar -zcvf %s tmp/" % filename)
    os.system("mv tmp.tar.gz %s" % path)
    os.system("tar -zxvf %s/%s --directory=%s --strip-components 1" % (path, filename, path))
    os.system("rm %s/%s" % (path, filename))
    os.system("mv %s/index.html %s/WEB-INF/templates/tickets.html" % (path, path))
<|endoftext|>


<commit_before>import os

import numpy
try:
    from PIL import Image
    available = True
except ImportError as e:
    available = False
    _import_error = e
import bisect
import io
import six
import threading
import zipfile

from chainer.dataset import dataset_mixin

def _read_image_as_array(path, dtype):
    f = Image.open(path)
    try:
        image = numpy.asarray(f, dtype=dtype)
    finally:
        # Only pillow >= 3.0 has 'close' method
        if hasattr(f, 'close'):
            f.close()
    return image

def _postprocess_image(image):
    if image.ndim == 2:
        # image is greyscale
        image = image[..., None]
    return image.transpose(2, 0, 1)

class ImageDataset(dataset_mixin.DatasetMixin):

    """Dataset of images built from a list of paths to image files.

    This dataset reads an external image file on every call of the
    :meth:`__getitem__` operator. The paths to the image to retrieve is given
    as either a list of strings or a text file that contains paths in distinct
    lines.

    Each image is automatically converted to arrays of shape
    ``channels, height, width``, where ``channels`` represents the number of
    channels in each pixel (e.g., 1 for grey-scale images, and 3 for RGB-color
    images).

    .. note::
       **This dataset requires the Pillow package being installed.** In order
       to use this dataset, install Pillow (e.g. by using the command ``pip
       install Pillow``). Be careful to prepare appropriate libraries for image
       formats you want to use (e.g. libpng for PNG images, and libjpeg for JPG
       images).

    .. warning::
       **You are responsible for preprocessing the images before feeding them
       to a model.** For example, if your dataset contains both RGB and
       grayscale images, make sure that you convert them to the same format.
       Otherwise you will get errors because the input dimensions are different
       for RGB and grayscale images.

    Args:
        paths (str or list of strs): If it is a string, it is a path to a text
            file that contains paths to images in distinct lines. If it is a
            list of paths, the ``i``-th element represents the path to the
            ``i``-th image. In both cases, each path is a relative one from the
            root path given by another argument.
        root (str): Root directory to retrieve images from.
        dtype: Data type of resulting image arrays.

    """

    def __init__(self, paths, root='.', dtype=numpy.float32):
        _check_pillow_availability()
        if isinstance(paths, six.string_types):
            with open(paths) as paths_file:
                paths = [path.strip() for path in paths_file]
        self._paths = paths
        self._root = root
        self._dtype = dtype

    def __len__(self):
        return len(self._paths)

    def get_example(self, i):
        path = os.path.join(self._root, self._paths[i])
        image = _read_image_as_array(path, self._dtype)

        return _postprocess_image(image)

class LabeledImageDataset(dataset_mixin.DatasetMixin):

    """Dataset of image and label pairs built from a list of paths and labels.

    This dataset reads an external image file like :class:`ImageDataset`. The
    difference from :class:`ImageDataset` is that this dataset also returns a
    label integer. The paths and labels are given as either a list of pairs or
    a text file contains paths/labels pairs in distinct lines. In the latter
    case, each path and corresponding label are separated by white spaces. This
    format is same as one used in Caffe.

    .. note::
       **This dataset requires the Pillow package being installed.** In order
       to use this dataset, install Pillow (e.g. by using the command ``pip
       install Pillow``). Be careful to prepare appropriate libraries for image
       formats you want to use (e.g. libpng for PNG images, and libjpeg for JPG
       images).

    .. warning::
       **You are responsible for preprocessing the images before feeding them
       to a model.** For example, if your dataset contains both RGB and
       grayscale images, make sure that you convert them to the same format.
       Otherwise you will get errors because the input dimensions are different
       for RGB and grayscale images.

    Args:
        pairs (str or list of tuples): If it is a string, it is a path to a
            text file that contains paths to images in distinct lines. If it is
            a list of pairs, the ``i``-th element represents a pair of the path
            to the ``i``-th image and the corresponding label. In both cases,
            each path is a relative one from the root path given by another
            argument.
        root (str): Root directory to retrieve images from.
        dtype: Data type of resulting image arrays.
        label_dtype: Data type of the labels.

    """

    def __init__(self, pairs, root='.', dtype=numpy.float32,
                 label_dtype=numpy.int32):
        _check_pillow_availability()
        if isinstance(pairs, six.string_types):
            pairs_path = pairs
            with open(pairs_path) as pairs_file:
                pairs = []
                for i, line in enumerate(pairs_file):
                    pair = line.strip().split()
                    if len(pair) != 2:
                        raise ValueError(
                            'invalid format at line {} in file {}'.format(
                                i, pairs_path))
                    pairs.append((pair[0], int(pair[1])))
        self._pairs = pairs
        self._root = root
        self._dtype = dtype
        self._label_dtype = label_dtype

    def __len__(self):
        return len(self._pairs)

    def get_example(self, i):
        path, int_label = self._pairs[i]
        full_path = os.path.join(self._root, path)
        image = _read_image_as_array(full_path, self._dtype)

        label = numpy.array(int_label, dtype=self._label_dtype)
        return _postprocess_image(image), label

class MultiZippedImageDataset(dataset_mixin.DatasetMixin):
    """Dataset of images built from a list of paths to zip files.

    This dataset reads an external image file in given zipfiles. The
    zipfiles shall contain only image files.
    This shall be able to replace ImageDataset and works better on NFS
    and other networked file systems. The user shall find good balance
    between zipfile size and number of zipfiles (e.g. granularity)

    Args:
        zipfilenames (list of strings): List of zipped archive filename.
        dtype: Data type of resulting image arrays.
    """

    def __init__(self, zipfilenames, dtype=numpy.float32):
        self._zfs = [ZippedImageDataset(fn, dtype) for fn in zipfilenames]
        self._zpaths_accumlens = [0]
        zplen = 0
        for zf in self._zfs:
            zplen += len(zf)
            self._zpaths_accumlens.append(zplen)

    def __len__(self):
        return self._zpaths_accumlens[-1]

    def get_example(self, i):
        tgt = bisect.bisect(self._zpaths_accumlens, i) - 1

        lidx = i - self._zpaths_accumlens[tgt]
        return self._zfs[tgt].get_example(lidx)

class ZippedImageDataset(dataset_mixin.DatasetMixin):
    """Dataset of images built from a zip file.

    This dataset reads an external image file in the given
    zipfile. The zipfile shall contain only image files.
    This shall be able to replace ImageDataset and works better on NFS
    and other networked file systems. If zipfile becomes too large you
    may consider ``MultiZippedImageDataset`` as a handy alternative.

    Known issue: pickle and unpickle on same process may cause race
    condition on ZipFile. Pickle of this class is expected to be sent
    to different processess via ChainerMN.

    Args:
        zipfilename (str): a string to point zipfile path
        dtype: Data type of resulting image arrays

    """

    def __init__(self, zipfilename, dtype=numpy.float32):
        self._zipfilename = zipfilename
        self._zf = zipfile.ZipFile(zipfilename)
        self._zf_pid = os.getpid()
        self._dtype = dtype
        self._paths = [x for x in self._zf.namelist() if not x.endswith('/')]
        self._lock = threading.Lock()

    def __len__(self):
        return len(self._paths)

    def __getstate__(self):
        d = self.__dict__.copy()
        d['_zf'] = None
        d['_lock'] = None
        return d

    def __setstate__(self, state):
        self.__dict__ = state
        self._lock = threading.Lock()

    def get_example(self, i):
        # PIL may seek() on the file -- zipfile won't support it
        with self._lock:
            if self._zf is None or self._zf_pid != os.getpid():
                self._zf_pid = os.getpid()
                self._zf = zipfile.ZipFile(self._zipfilename)
            image_file_mem = self._zf.read(self._paths[i])
        image_file = io.BytesIO(image_file_mem)
        image = _read_image_as_array(image_file, self._dtype)
        return _postprocess_image(image)

def _check_pillow_availability():
    if not available:
        raise ImportError('PIL cannot be loaded. Install Pillow!\n'
                          'The actual import error is as follows:\n' +
                          str(_import_error))

<commit_msg>removed unneccessary blank line<commit_after>import os

import numpy
try:
    from PIL import Image
    available = True
except ImportError as e:
    available = False
    _import_error = e
import bisect
import io
import six
import threading
import zipfile

from chainer.dataset import dataset_mixin

def _read_image_as_array(path, dtype):
    f = Image.open(path)
    try:
        image = numpy.asarray(f, dtype=dtype)
    finally:
        # Only pillow >= 3.0 has 'close' method
        if hasattr(f, 'close'):
            f.close()
    return image

def _postprocess_image(image):
    if image.ndim == 2:
        # image is greyscale
        image = image[..., None]
    return image.transpose(2, 0, 1)

class ImageDataset(dataset_mixin.DatasetMixin):

    """Dataset of images built from a list of paths to image files.

    This dataset reads an external image file on every call of the
    :meth:`__getitem__` operator. The paths to the image to retrieve is given
    as either a list of strings or a text file that contains paths in distinct
    lines.

    Each image is automatically converted to arrays of shape
    ``channels, height, width``, where ``channels`` represents the number of
    channels in each pixel (e.g., 1 for grey-scale images, and 3 for RGB-color
    images).

    .. note::
       **This dataset requires the Pillow package being installed.** In order
       to use this dataset, install Pillow (e.g. by using the command ``pip
       install Pillow``). Be careful to prepare appropriate libraries for image
       formats you want to use (e.g. libpng for PNG images, and libjpeg for JPG
       images).

    .. warning::
       **You are responsible for preprocessing the images before feeding them
       to a model.** For example, if your dataset contains both RGB and
       grayscale images, make sure that you convert them to the same format.
       Otherwise you will get errors because the input dimensions are different
       for RGB and grayscale images.

    Args:
        paths (str or list of strs): If it is a string, it is a path to a text
            file that contains paths to images in distinct lines. If it is a
            list of paths, the ``i``-th element represents the path to the
            ``i``-th image. In both cases, each path is a relative one from the
            root path given by another argument.
        root (str): Root directory to retrieve images from.
        dtype: Data type of resulting image arrays.

    """

    def __init__(self, paths, root='.', dtype=numpy.float32):
        _check_pillow_availability()
        if isinstance(paths, six.string_types):
            with open(paths) as paths_file:
                paths = [path.strip() for path in paths_file]
        self._paths = paths
        self._root = root
        self._dtype = dtype

    def __len__(self):
        return len(self._paths)

    def get_example(self, i):
        path = os.path.join(self._root, self._paths[i])
        image = _read_image_as_array(path, self._dtype)

        return _postprocess_image(image)

class LabeledImageDataset(dataset_mixin.DatasetMixin):

    """Dataset of image and label pairs built from a list of paths and labels.

    This dataset reads an external image file like :class:`ImageDataset`. The
    difference from :class:`ImageDataset` is that this dataset also returns a
    label integer. The paths and labels are given as either a list of pairs or
    a text file contains paths/labels pairs in distinct lines. In the latter
    case, each path and corresponding label are separated by white spaces. This
    format is same as one used in Caffe.

    .. note::
       **This dataset requires the Pillow package being installed.** In order
       to use this dataset, install Pillow (e.g. by using the command ``pip
       install Pillow``). Be careful to prepare appropriate libraries for image
       formats you want to use (e.g. libpng for PNG images, and libjpeg for JPG
       images).

    .. warning::
       **You are responsible for preprocessing the images before feeding them
       to a model.** For example, if your dataset contains both RGB and
       grayscale images, make sure that you convert them to the same format.
       Otherwise you will get errors because the input dimensions are different
       for RGB and grayscale images.

    Args:
        pairs (str or list of tuples): If it is a string, it is a path to a
            text file that contains paths to images in distinct lines. If it is
            a list of pairs, the ``i``-th element represents a pair of the path
            to the ``i``-th image and the corresponding label. In both cases,
            each path is a relative one from the root path given by another
            argument.
        root (str): Root directory to retrieve images from.
        dtype: Data type of resulting image arrays.
        label_dtype: Data type of the labels.

    """

    def __init__(self, pairs, root='.', dtype=numpy.float32,
                 label_dtype=numpy.int32):
        _check_pillow_availability()
        if isinstance(pairs, six.string_types):
            pairs_path = pairs
            with open(pairs_path) as pairs_file:
                pairs = []
                for i, line in enumerate(pairs_file):
                    pair = line.strip().split()
                    if len(pair) != 2:
                        raise ValueError(
                            'invalid format at line {} in file {}'.format(
                                i, pairs_path))
                    pairs.append((pair[0], int(pair[1])))
        self._pairs = pairs
        self._root = root
        self._dtype = dtype
        self._label_dtype = label_dtype

    def __len__(self):
        return len(self._pairs)

    def get_example(self, i):
        path, int_label = self._pairs[i]
        full_path = os.path.join(self._root, path)
        image = _read_image_as_array(full_path, self._dtype)

        label = numpy.array(int_label, dtype=self._label_dtype)
        return _postprocess_image(image), label

class MultiZippedImageDataset(dataset_mixin.DatasetMixin):
    """Dataset of images built from a list of paths to zip files.

    This dataset reads an external image file in given zipfiles. The
    zipfiles shall contain only image files.
    This shall be able to replace ImageDataset and works better on NFS
    and other networked file systems. The user shall find good balance
    between zipfile size and number of zipfiles (e.g. granularity)

    Args:
        zipfilenames (list of strings): List of zipped archive filename.
        dtype: Data type of resulting image arrays.
    """

    def __init__(self, zipfilenames, dtype=numpy.float32):
        self._zfs = [ZippedImageDataset(fn, dtype) for fn in zipfilenames]
        self._zpaths_accumlens = [0]
        zplen = 0
        for zf in self._zfs:
            zplen += len(zf)
            self._zpaths_accumlens.append(zplen)

    def __len__(self):
        return self._zpaths_accumlens[-1]

    def get_example(self, i):
        tgt = bisect.bisect(self._zpaths_accumlens, i) - 1

        lidx = i - self._zpaths_accumlens[tgt]
        return self._zfs[tgt].get_example(lidx)

class ZippedImageDataset(dataset_mixin.DatasetMixin):
    """Dataset of images built from a zip file.

    This dataset reads an external image file in the given
    zipfile. The zipfile shall contain only image files.
    This shall be able to replace ImageDataset and works better on NFS
    and other networked file systems. If zipfile becomes too large you
    may consider ``MultiZippedImageDataset`` as a handy alternative.

    Known issue: pickle and unpickle on same process may cause race
    condition on ZipFile. Pickle of this class is expected to be sent
    to different processess via ChainerMN.

    Args:
        zipfilename (str): a string to point zipfile path
        dtype: Data type of resulting image arrays

    """

    def __init__(self, zipfilename, dtype=numpy.float32):
        self._zipfilename = zipfilename
        self._zf = zipfile.ZipFile(zipfilename)
        self._zf_pid = os.getpid()
        self._dtype = dtype
        self._paths = [x for x in self._zf.namelist() if not x.endswith('/')]
        self._lock = threading.Lock()

    def __len__(self):
        return len(self._paths)

    def __getstate__(self):
        d = self.__dict__.copy()
        d['_zf'] = None
        d['_lock'] = None
        return d

    def __setstate__(self, state):
        self.__dict__ = state
        self._lock = threading.Lock()

    def get_example(self, i):
        # PIL may seek() on the file -- zipfile won't support it
        with self._lock:
            if self._zf is None or self._zf_pid != os.getpid():
                self._zf_pid = os.getpid()
                self._zf = zipfile.ZipFile(self._zipfilename)
            image_file_mem = self._zf.read(self._paths[i])
        image_file = io.BytesIO(image_file_mem)
        image = _read_image_as_array(image_file, self._dtype)
        return _postprocess_image(image)

def _check_pillow_availability():
    if not available:
        raise ImportError('PIL cannot be loaded. Install Pillow!\n'
                          'The actual import error is as follows:\n' +
                          str(_import_error))
<|endoftext|>


<commit_before>import numpy
import six

import chainer
from chainer import cuda
from chainer import function_node
from chainer.utils import type_check

class Vstack(function_node.FunctionNode):

    """Concatenate multiple tensors vertically (row wise)."""

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() > 0)

        ndim = type_check.eval(in_types[0].ndim)
        for i in six.moves.range(1, type_check.eval(in_types.size())):
            type_check.expect(
                in_types[0].dtype == in_types[i].dtype,
                in_types[0].ndim == in_types[i].ndim,
            )
            if ndim <= 1:
                type_check.expect(in_types[0].shape == in_types[i].shape)
                continue
            for d in six.moves.range(1, ndim):
                type_check.expect(in_types[0].shape[d] == in_types[i].shape[d])

    def forward(self, xs):
        self._in_shapes = [x.shape for x in xs]
        xp = cuda.get_array_module(*xs)
        return xp.vstack(xs),

    def backward(self, indexes, grad_outputs):
        gy, = grad_outputs
        ndim = len(self.inputs[0].shape)
        if len(self.inputs) == 1:
            if ndim <= 1:
                return gy.reshape(self.inputs[0].shape),
            return gy,

        if ndim <= 1:
            gx = chainer.functions.split_axis(gy, len(self.inputs), 0)
            return [g.reshape(self.inputs[0].shape) for g in gx]

        sizes = numpy.array([x.shape[0] for x in self.inputs[:-1]]).cumsum()
        return chainer.functions.split_axis(gy, sizes, 0)

def vstack(xs):
    """Concatenate variables vertically (row wise).

    Args:
        xs (list of :class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Input variables to be concatenated. The variables must have the
            same ``ndim``. When the variables have the second axis (i.e.
            :math:`ndim \\geq 2`), the variables must have the same shape
            along all but the first axis. When the variables do not have the
            second axis(i.e. :math:`ndim < 2`), the variables must have the
            same shape.

    Returns:
        ~chainer.Variable:
            Output variable. When the input variables have the second axis
            (i.e. :math:`ndim \\geq 2`), the shapes of inputs and output are
            the same along all but the first axis. The length of first axis
            is the sum of the lengths of inputs' first axis.
            When the variables do not have the second axis (i.e.
            :math:`ndim < 2`), the shape of output is ``(2, N)`` (``N`` is the
            size of the input variable).

    .. admonition:: Example

        >>> x1 = np.array((1, 2, 3))
        >>> x1.shape
        (3,)
        >>> x2 = np.array((2, 3, 4))
        >>> x2.shape
        (3,)
        >>> y = F.vstack((x1, x2))
        >>> y.shape
        (2, 3)
        >>> y.data
        array([[1, 2, 3],
               [2, 3, 4]])
        >>> x1 = np.arange(0, 12).reshape(3, 4)
        >>> x1.shape
        (3, 4)
        >>> x1
        array([[ 0,  1,  2,  3],
               [ 4,  5,  6,  7],
               [ 8,  9, 10, 11]])
        >>> x2 = np.arange(12, 20).reshape(2, 4)
        >>> x2.shape
        (2, 4)
        >>> x2
        array([[12, 13, 14, 15],
               [16, 17, 18, 19]])
        >>> y = F.vstack([x1, x2])
        >>> y.shape
        (5, 4)
        >>> y.data
        array([[ 0,  1,  2,  3],
               [ 4,  5,  6,  7],
               [ 8,  9, 10, 11],
               [12, 13, 14, 15],
               [16, 17, 18, 19]])

    """

    return Vstack().apply((xs))[0]
<commit_msg>Remove unused code<commit_after>import numpy
import six

import chainer
from chainer import cuda
from chainer import function_node
from chainer.utils import type_check

class Vstack(function_node.FunctionNode):

    """Concatenate multiple tensors vertically (row wise)."""

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() > 0)

        ndim = type_check.eval(in_types[0].ndim)
        for i in six.moves.range(1, type_check.eval(in_types.size())):
            type_check.expect(
                in_types[0].dtype == in_types[i].dtype,
                in_types[0].ndim == in_types[i].ndim,
            )
            if ndim <= 1:
                type_check.expect(in_types[0].shape == in_types[i].shape)
                continue
            for d in six.moves.range(1, ndim):
                type_check.expect(in_types[0].shape[d] == in_types[i].shape[d])

    def forward(self, xs):
        xp = cuda.get_array_module(*xs)
        return xp.vstack(xs),

    def backward(self, indexes, grad_outputs):
        gy, = grad_outputs
        ndim = len(self.inputs[0].shape)
        if len(self.inputs) == 1:
            if ndim <= 1:
                return gy.reshape(self.inputs[0].shape),
            return gy,

        if ndim <= 1:
            gxs = chainer.functions.split_axis(gy, len(self.inputs), 0)
            return [gx.reshape(self.inputs[0].shape) for gx in gxs]

        sizes = numpy.array([x.shape[0] for x in self.inputs[:-1]]).cumsum()
        return chainer.functions.split_axis(gy, sizes, 0)

def vstack(xs):
    """Concatenate variables vertically (row wise).

    Args:
        xs (list of :class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Input variables to be concatenated. The variables must have the
            same ``ndim``. When the variables have the second axis (i.e.
            :math:`ndim \\geq 2`), the variables must have the same shape
            along all but the first axis. When the variables do not have the
            second axis(i.e. :math:`ndim < 2`), the variables must have the
            same shape.

    Returns:
        ~chainer.Variable:
            Output variable. When the input variables have the second axis
            (i.e. :math:`ndim \\geq 2`), the shapes of inputs and output are
            the same along all but the first axis. The length of first axis
            is the sum of the lengths of inputs' first axis.
            When the variables do not have the second axis (i.e.
            :math:`ndim < 2`), the shape of output is ``(2, N)`` (``N`` is the
            size of the input variable).

    .. admonition:: Example

        >>> x1 = np.array((1, 2, 3))
        >>> x1.shape
        (3,)
        >>> x2 = np.array((2, 3, 4))
        >>> x2.shape
        (3,)
        >>> y = F.vstack((x1, x2))
        >>> y.shape
        (2, 3)
        >>> y.data
        array([[1, 2, 3],
               [2, 3, 4]])
        >>> x1 = np.arange(0, 12).reshape(3, 4)
        >>> x1.shape
        (3, 4)
        >>> x1
        array([[ 0,  1,  2,  3],
               [ 4,  5,  6,  7],
               [ 8,  9, 10, 11]])
        >>> x2 = np.arange(12, 20).reshape(2, 4)
        >>> x2.shape
        (2, 4)
        >>> x2
        array([[12, 13, 14, 15],
               [16, 17, 18, 19]])
        >>> y = F.vstack([x1, x2])
        >>> y.shape
        (5, 4)
        >>> y.data
        array([[ 0,  1,  2,  3],
               [ 4,  5,  6,  7],
               [ 8,  9, 10, 11],
               [12, 13, 14, 15],
               [16, 17, 18, 19]])

    """

    return Vstack().apply((xs))[0]
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, unicode_literals

from datetime import datetime, timedelta
from flask import Response
from sqlalchemy import and_
from sqlalchemy.orm import joinedload, subqueryload

from changes.api.base import APIView
from changes.config import db
from changes.constants import Status
from changes.models import Project, AggregateTestGroup, TestGroup, Job

class ProjectTestIndexAPIView(APIView):
    def _get_project(self, project_id):
        queryset = Project.query.options(
            joinedload(Project.repository),
        )

        project = queryset.filter_by(slug=project_id).first()
        if project is None:
            project = queryset.get(project_id)
        return project

    def get(self, project_id):
        project = self._get_project(project_id)
        if not project:
            return Response(status=404)

        cutoff = datetime.utcnow() - timedelta(days=3)

        latest_job = Job.query.options(
            joinedload(Job.project),
            joinedload(Job.author),
        ).filter(
            Job.revision_sha != None,  # NOQA
            Job.patch_id == None,
            Job.project == project,
            Job.status == Status.finished,
        ).order_by(
            Job.date_created.desc(),
        ).first()

        if latest_job:
            test_list = db.session.query(AggregateTestGroup, TestGroup).options(
                subqueryload(AggregateTestGroup.first_job),
                subqueryload(AggregateTestGroup.parent),
                subqueryload(TestGroup.parent),
            ).join(
                TestGroup, and_(
                    TestGroup.job_id == latest_job.id,
                    TestGroup.name_sha == AggregateTestGroup.name_sha,
                )
            ).filter(
                AggregateTestGroup.parent_id == None,  # NOQA: we have to use == here
                AggregateTestGroup.project_id == project.id,
                Job.date_created > cutoff,
            ).order_by(TestGroup.duration.desc())

            results = []
            for agg, group in test_list:
                agg.last_testgroup = group
                results.append(agg)
        else:
            results = []

        context = {
            'tests': results,
        }

        return self.respond(context)
<commit_msg>Remove invalid job join<commit_after>from __future__ import absolute_import, division, unicode_literals

from datetime import datetime, timedelta
from flask import Response
from sqlalchemy import and_
from sqlalchemy.orm import joinedload, subqueryload

from changes.api.base import APIView
from changes.config import db
from changes.constants import Status
from changes.models import Project, AggregateTestGroup, TestGroup, Job

class ProjectTestIndexAPIView(APIView):
    def _get_project(self, project_id):
        queryset = Project.query.options(
            joinedload(Project.repository),
        )

        project = queryset.filter_by(slug=project_id).first()
        if project is None:
            project = queryset.get(project_id)
        return project

    def get(self, project_id):
        project = self._get_project(project_id)
        if not project:
            return Response(status=404)

        latest_job = Job.query.options(
            subqueryload(Job.project),
            subqueryload(Job.author),
        ).filter(
            Job.revision_sha != None,  # NOQA
            Job.patch_id == None,
            Job.project == project,
            Job.status == Status.finished,
        ).order_by(
            Job.date_created.desc(),
        ).limit(1).first()

        if latest_job:
            test_list = db.session.query(AggregateTestGroup, TestGroup).options(
                subqueryload(AggregateTestGroup.first_job),
                subqueryload(AggregateTestGroup.parent),
                subqueryload(TestGroup.parent),
            ).join(
                TestGroup, and_(
                    TestGroup.job_id == latest_job.id,
                    TestGroup.name_sha == AggregateTestGroup.name_sha,
                )
            ).filter(
                AggregateTestGroup.parent_id == None,  # NOQA: we have to use == here
                AggregateTestGroup.project_id == project.id,
            ).order_by(TestGroup.duration.desc())

            results = []
            for agg, group in test_list:
                agg.last_testgroup = group
                results.append(agg)
        else:
            results = []

        context = {
            'tests': results,
        }

        return self.respond(context)
<|endoftext|>


<commit_before>import logging
from django.template.response import SimpleTemplateResponse
from django.db import IntegrityError
from chatterbox.models import Activity
from chatterbox.exceptions import KeyInvalidationException

log = logging.getLogger(__name__)

def cycle(count):
    current = 0

    while 1:
        yield current
        current = current + 1

        if current >= count:
            current = 0

class KeyManager(object):
    def __init__(self, keys):
        self.keys = keys
        self._cycle = cycle(len(keys))

        try:
            self.current_key = keys[0]
            self.current_index = next(self._cycle)
        except IndexError:
            self.current_key = None
            self.current_index = -1

    @property
    def api(self):
        return self.current_key.api

    def invalidate_current_key(self):
        index = next(self._cycle)

        if index == self.current_index:
            log.debug("Unable to invalidate current key. Next key is current key")
            raise KeyInvalidationException

        log.debug("Previous key has been invalidated, switching "
                  "to next available key")

        self.current_key = self.keys[index]
        self.current_index = index

class Collector(object):
    form = None
    template = None
    activity_from_dict = lambda: None

    def __init__(self):
        self.keys = []
        self.key_manager = None

    @property
    def keys(self):
        return self.key_manager.keys

    @keys.setter
    def keys(self, value):
        self.key_manager = KeyManager(value)

    @property
    def api(self):
        return self.key_manager.api

    def action(self, job):
        raise NotImplementedError("Subclasses must implement the \"action\" method")

    def render(self):
        log.debug("Rendering Collector")
        form = None
        template = self.template

        if self.form:
            log.debug("Initializing Collector Form")
            form = self.form(prefix="data")

        if template and form:
            log.debug("Initializing Collector Template")
            context = {"form": form}
            view = SimpleTemplateResponse(template, context)
            log.debug("Rendering form with template")
            return view.render()

        elif template is None and form:
            log.debug("Rendering form as table")
            return form.as_table()

        else:
            log.debug("No template or form to render")
            return ""

    def create_or_get_activity_from_dict(self, data):
        log.debug("Creating or getting activity from dict: %s", data)
        activity = self.activity_from_dict.__func__(data)
        # try and save the object after setting the ID, if there is error
        # it's because the object is not unique
        try:
            activity.save()
        except IntegrityError:
            # it already exists..but is it from another job?
            log.debug("Activity exists, fetching existing")
            activity = Activity.objects.get(object_id=activity.object_id)

        return activity

    def register_activity(self, activity, job):
        log.debug("Registering activity: %s for job: %s", activity, job)
        count = activity.job.filter(id=job.id).count()

        if count:
            log.error("Job: %s already associated with activity: %s", job, activity)
            raise Exception("Job already associated with activity")

        activity.job.add(job)
<commit_msg>removes empty array init for keys<commit_after>import logging
from django.template.response import SimpleTemplateResponse
from django.db import IntegrityError
from chatterbox.models import Activity
from chatterbox.exceptions import KeyInvalidationException

log = logging.getLogger(__name__)

def cycle(count):
    current = 0

    while 1:
        yield current
        current = current + 1

        if current >= count:
            current = 0

class KeyManager(object):
    def __init__(self, keys):
        self.keys = keys
        self._cycle = cycle(len(keys))

        self.current_key = keys[0]
        self.current_index = next(self._cycle)

    @property
    def api(self):
        return self.current_key.api

    def invalidate_current_key(self):
        index = next(self._cycle)

        if index == self.current_index:
            log.debug("Unable to invalidate current key. Next key is current key")
            raise KeyInvalidationException

        log.debug("Previous key has been invalidated, switching "
                  "to next available key")

        self.current_key = self.keys[index]
        self.current_index = index

class Collector(object):
    form = None
    template = None
    activity_from_dict = lambda: None

    def __init__(self):
        self.key_manager = None

    @property
    def keys(self):
        return self.key_manager.keys

    @keys.setter
    def keys(self, value):
        self.key_manager = KeyManager(value)

    @property
    def api(self):
        return self.key_manager.api

    def action(self, job):
        raise NotImplementedError("Subclasses must implement the \"action\" method")

    def render(self):
        log.debug("Rendering Collector")
        form = None
        template = self.template

        if self.form:
            log.debug("Initializing Collector Form")
            form = self.form(prefix="data")

        if template and form:
            log.debug("Initializing Collector Template")
            context = {"form": form}
            view = SimpleTemplateResponse(template, context)
            log.debug("Rendering form with template")
            return view.render()

        elif template is None and form:
            log.debug("Rendering form as table")
            return form.as_table()

        else:
            log.debug("No template or form to render")
            return ""

    def create_or_get_activity_from_dict(self, data):
        log.debug("Creating or getting activity from dict: %s", data)
        activity = self.activity_from_dict.__func__(data)
        # try and save the object after setting the ID, if there is error
        # it's because the object is not unique
        try:
            activity.save()
        except IntegrityError:
            # it already exists..but is it from another job?
            log.debug("Activity exists, fetching existing")
            activity = Activity.objects.get(object_id=activity.object_id)

        return activity

    def register_activity(self, activity, job):
        log.debug("Registering activity: %s for job: %s", activity, job)
        count = activity.job.filter(id=job.id).count()

        if count:
            log.error("Job: %s already associated with activity: %s", job, activity)
            raise Exception("Job already associated with activity")

        activity.job.add(job)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import glob
import os
import re
import shutil
import tempfile
import urlparse

import pyauto_functional  # Must be imported before pyauto
import pyauto
import test_utils

class OmniboxTest(pyauto.PyUITest):
  """TestCase for Omnibox."""

  def Debug(self):
    """Test method for experimentation.

    This method will not run automatically.
    """
    import pprint
    import time
    pp = pprint.PrettyPrinter(indent=2)
    while True:
      pp.pprint(self.GetOmniboxInfo().omniboxdict)
      time.sleep(1)

  def testFocusOnStartup(self):
    """Verify that omnibox has focus on startup."""
    self.WaitUntilOmniboxReadyHack()
    self.assertTrue(self.GetOmniboxInfo().Properties('has_focus'))

  def testHistoryResult(self):
    """Verify that omnibox can fetch items from history."""
    url = self.GetFileURLForDataPath('title2.html')
    title = 'Title Of Awesomeness'
    self.AppendTab(pyauto.GURL(url))
    def _VerifyHistoryResult(query_list, description, windex=0):
      """Verify result matching given description for given list of queries."""
      for query_text in query_list:
        matches = test_utils.GetOmniboxMatchesFor(self,
            query_text, windex=windex, attr_dict={'description': description})
        self.assertTrue(matches)
        self.assertEqual(1, len(matches))
        item = matches[0]
        self.assertEqual(url, item['destination_url'])
    # Query using URL & title
    _VerifyHistoryResult([url, title], title)
    # Verify results in another tab
    self.AppendTab(pyauto.GURL())
    _VerifyHistoryResult([url, title], title)
    # Verify results in another window
    self.OpenNewBrowserWindow(True)
    self.WaitUntilOmniboxReadyHack(windex=1)
    _VerifyHistoryResult([url, title], title, windex=1)
    # Verify results in an incognito window
    self.RunCommand(pyauto.IDC_NEW_INCOGNITO_WINDOW)
    self.WaitUntilOmniboxReadyHack(windex=2)
    _VerifyHistoryResult([url, title], title, windex=2)

  def _VerifyOmniboxURLMatches(self, url, description, windex=0):
    """Verify URL match results from the Omnibox.

    Args:
      url: the url to use
      description: the string description within history page and google search
                   to match against
      windex: the window index to work on. Defaults to 0 (first window)
    """
    matches_description = test_utils.GetOmniboxMatchesFor(self,
        url, windex=windex, attr_dict={'description': description})
    self.assertEqual(1, len(matches_description))
    if description == 'Google Search':
      self.assertTrue(re.match('http://www.google.com/search.+',
                               matches_description[0]['destination_url']))
    else:
      self.assertEqual(url, matches_description[0]['destination_url'])

  def testFetchHistoryResultItems(self):
    """Verify omnibox fetches history items in second tab, win and Incognito."""
    url = self.GetFileURLForDataPath('title2.html')
    title = 'Title Of Awesomeness'
    desc = 'Google Search'
    # fetch history page item in the second tab.
    self.AppendTab(pyauto.GURL(url))
    self._VerifyOmniboxURLMatches(url, title)
    # fetch history page items in the second window.
    self.OpenNewBrowserWindow(True)
    self.NavigateToURL(url, 1, 0)
    self._VerifyOmniboxURLMatches(url, title, windex=1)
    # fetch google search items in Incognito window.
    self.RunCommand(pyauto.IDC_NEW_INCOGNITO_WINDOW)
    self.NavigateToURL(url, 2, 0)
    self._VerifyOmniboxURLMatches(url, desc, windex=2)

  def testSelect(self):
    """Verify omnibox popup selection."""
    url1 = self.GetFileURLForDataPath('title2.html')
    url2 = self.GetFileURLForDataPath('title1.html')
    title1 = 'Title Of Awesomeness'
    self.NavigateToURL(url1)
    self.NavigateToURL(url2)
    matches = test_utils.GetOmniboxMatchesFor(self, 'file://')
    self.assertTrue(matches)
    # Find the index of match for url1
    index = None
    for i, match in enumerate(matches):
      if match['description'] == title1:
        index = i
    self.assertTrue(index is not None)
    self.OmniboxMovePopupSelection(index)  # Select url1 line in popup
    self.assertEqual(url1, self.GetOmniboxInfo().Text())
    self.OmniboxAcceptInput()
    self.assertEqual(title1, self.GetActiveTabTitle())

  def testGoogleSearch(self):
    """Verify Google search item in omnibox results."""
    search_text = 'hello world'
    verify_str = 'Google Search'
    url_re = 'http://www.google.com/search\?.*q=hello\+world.*'
    matches_description = test_utils.GetOmniboxMatchesFor(self,
        search_text, attr_dict={'description': verify_str})
    self.assertTrue(matches_description)
    # There should be a least one entry with the description Google. Suggest
    # results may end up having 'Google Search' in them, so use >=.
    self.assertTrue(len(matches_description) >= 1)
    item = matches_description[0]
    self.assertTrue(re.search(url_re, item['destination_url']))
    self.assertEqual('search-what-you-typed', item['type'])

  def testInlinAutoComplete(self):
    """Verify inline autocomplete for a pre-visited url."""
    self.NavigateToURL('http://www.google.com')
    matches = test_utils.GetOmniboxMatchesFor(self, 'goog')
    self.assertTrue(matches)
    # Omnibox should suggest auto completed url as the first item
    matches_description = matches[0]
    self.assertTrue('www.google.com' in matches_description['contents'])
    self.assertEqual('history-url', matches_description['type'])
    # The url should be inline-autocompleted in the omnibox
    self.assertTrue('google.com' in self.GetOmniboxInfo().Text())

  def testCrazyFilenames(self):
    """Test omnibox query with filenames containing special chars.

    The files are created on the fly and cleaned after use.
    """
    filename = os.path.join(self.DataDir(), 'downloads', 'crazy_filenames.txt')
    zip_names = self.EvalDataFrom(filename)
    # We got .zip filenames. Change them to .html
    crazy_filenames = [x.replace('.zip', '.html') for x in zip_names]
    title = 'given title'

    def _CreateFile(name):
      """Create the given html file."""
      fp = open(name, 'w')  # name could be unicode
      print >>fp, '<title>%s</title>' % title
      print >>fp, 'This is a junk file named
%s
' % repr(name)
      print >>fp, ''
      fp.close()

    crazy_fileurls = []
    # Temp dir for hosting crazy filenames.
    temp_dir = tempfile.mkdtemp(prefix='omnibox')
    # Windows has a dual nature dealing with unicode filenames.
    # While the files are internally saved as unicode, there's a non-unicode
    # aware API that returns a locale-dependent coding on the true unicode
    # filenames.  This messes up things.
    # Filesystem-interfacing functions like os.listdir() need to
    # be given unicode strings to "do the right thing" on win.
    # Ref: http://boodebr.org/main/python/all-about-python-and-unicode
    try:
      for filename in crazy_filenames:  # filename is unicode.
        file_path = os.path.join(temp_dir, filename.encode('utf-8'))
        _CreateFile(os.path.join(temp_dir, filename))
        file_url = self.GetFileURLForPath(file_path)
        crazy_fileurls.append(file_url)
        self.NavigateToURL(file_url)

      # Verify omnibox queries.
      for file_url in crazy_fileurls:
        matches = test_utils.GetOmniboxMatchesFor(self,
            file_url, attr_dict={'type': 'url-what-you-typed',
                                 'description': title})
        self.assertTrue(matches)
        self.assertEqual(1, len(matches))
        self.assertTrue(os.path.basename(file_url) in
                        matches[0]['destination_url'])
    finally:
      shutil.rmtree(unicode(temp_dir))  # unicode so that win treats nicely.

  def testSuggest(self):
    """Verify suggested results in omnibox."""
    matches = test_utils.GetOmniboxMatchesFor(self, 'apple')
    self.assertTrue(matches)
    self.assertTrue([x for x in matches if x['type'] == 'search-suggest'])

  def testDifferentTypesOfResults(self):
    """Verify different types of results from omnibox.

    This includes history result, bookmark result, suggest results.
    """
    url = 'http://www.google.com/'
    title = 'Google'
    search_string = 'google'
    self.AddBookmarkURL(  # Add a bookmark
        self.GetBookmarkModel().BookmarkBar()['id'], 0, title, url)
    self.NavigateToURL(url)  # Build up history
    matches = test_utils.GetOmniboxMatchesFor(self, search_string)
    self.assertTrue(matches)
    # Verify starred result (indicating bookmarked url)
    self.assertTrue([x for x in matches if x['starred'] == True])
    for item_type in ('history-url', 'search-what-you-typed',
                      'search-suggest',):
      self.assertTrue([x for x in matches if x['type'] == item_type])

  def testSuggestPref(self):
    """Verify no suggests for omnibox when suggested-services disabled."""
    search_string = 'apple'
    self.assertTrue(self.GetPrefsInfo().Prefs(pyauto.kSearchSuggestEnabled))
    matches = test_utils.GetOmniboxMatchesFor(self, search_string)
    self.assertTrue(matches)
    self.assertTrue([x for x in matches if x['type'] == 'search-suggest'])
    # Disable suggest-service
    self.SetPrefs(pyauto.kSearchSuggestEnabled, False)
    self.assertFalse(self.GetPrefsInfo().Prefs(pyauto.kSearchSuggestEnabled))
    matches = test_utils.GetOmniboxMatchesFor(self, search_string)
    self.assertTrue(matches)
    # Verify there are no suggest results
    self.assertFalse([x for x in matches if x['type'] == 'search-suggest'])

  def testAutoCompleteForSearch(self):
    """Verify omnibox autocomplete for search."""
    search_string = 'youtu'
    verify_string = 'youtube'
    matches = test_utils.GetOmniboxMatchesFor(self, search_string)
    # retrieve last contents element.
    matches_description = matches[-1]['contents'].split()
    self.assertEqual(verify_string, matches_description[0])

  def _CheckBookmarkResultForVariousInputs(self, url, title, windex=0):
    """Check if we get the Bookmark for complete and partial inputs."""
    # Check if the complete URL would get the bookmark.
    url_matches = test_utils.GetOmniboxMatchesFor(self, url, windex=windex)
    self._VerifyHasBookmarkResult(url_matches)
    # Check if the complete title would get the bookmark.
    title_matches = test_utils.GetOmniboxMatchesFor(self, title, windex=windex)
    self._VerifyHasBookmarkResult(title_matches)
    # Check if the partial URL would get the bookmark.
    split_url = urlparse.urlsplit(url)
    partial_url = test_utils.GetOmniboxMatchesFor(self,
        split_url.scheme, windex=windex)
    self._VerifyHasBookmarkResult(partial_url)
    # Check if the partial title would get the bookmark.
    split_title = title.split()
    search_term = split_title[len(split_title) - 1]
    partial_title = test_utils.GetOmniboxMatchesFor(self,
        search_term, windex=windex)
    self._VerifyHasBookmarkResult(partial_title)

  def _GotContentHistory(self, search_text, url):
    """Determines if omnibox returns a previously visited page for given
       search text
    """
    # Omnibox doesn't change results if searching the same text repeatedly.
    # So setting '' in omnibox before the next repeated search.
    self.SetOmniboxText('')
    matches = test_utils.GetOmniboxMatchesFor(self, search_text)
    matches_description = [x for x in matches if x['destination_url'] == url]
    return 1 == len(matches_description)

  def testContentHistory(self):
    """Verify omnibox results when entering page content

    Test verifies that visited page shows up in omnibox on entering page
    content.
    """
    url = self.GetFileURLForPath(
        os.path.join(self.DataDir(), 'find_in_page', 'largepage.html'))
    self.NavigateToURL(url)
    self.assertTrue(self.WaitUntil(
        lambda: self._GotContentHistory('British throne', url)))

  def testOmniboxSearchHistory(self):
    """Verify Page navigation/search from omnibox are added to the history."""
    url = self.GetFileURLForDataPath('title2.html')
    self.NavigateToURL(url)
    self.AppendTab(pyauto.GURL('about:blank'))
    self.SetOmniboxText('java')
    self.WaitUntilOmniboxQueryDone()
    self.OmniboxAcceptInput()
    history = self.GetHistoryInfo().History()
    self.assertEqual(2, len(history))
    self.assertEqual(url, history[1]['url'])
    self.assertEqual('java - Google Search', history[0]['title'])

  def _VerifyHasBookmarkResult(self, matches):
    """Verify that we have a bookmark result."""
    matches_starred = [result for result in matches if result['starred']]
    self.assertTrue(matches_starred)
    self.assertEqual(1, len(matches_starred))

  def testBookmarkResultInNewTabAndWindow(self):
    """Verify that omnibox can recognize a bookmark within search options
    in new tabs and windows."""
    url = self.GetFileURLForDataPath('title2.html')
    self.NavigateToURL(url)
    title = 'This is Awesomeness'
    bookmarks = self.GetBookmarkModel()
    bar_id = bookmarks.BookmarkBar()['id']
    self.AddBookmarkURL(bar_id, 0, title, url)
    bookmarks = self.GetBookmarkModel()
    nodes = bookmarks.FindByTitle(title)
    self.AppendTab(pyauto.GURL(url))
    self._CheckBookmarkResultForVariousInputs(url, title)
    self.OpenNewBrowserWindow(True)
    self.assertEqual(2, self.GetBrowserWindowCount())
    self.NavigateToURL(url, 1, 0)
    self._CheckBookmarkResultForVariousInputs(url, title, windex=1)
    self.RunCommand(pyauto.IDC_NEW_INCOGNITO_WINDOW)
    self.assertEqual(3, self.GetBrowserWindowCount())
    self.NavigateToURL(url, 2, 0)
    self._CheckBookmarkResultForVariousInputs(url, title, windex=2)

if __name__ == '__main__':
  pyauto_functional.Main()
<commit_msg>Added testAppSearch and testAutoCompleteForNonAsciiSearch new pyauto tests in omnibox.py<commit_after>#!/usr/bin/python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import glob
import os
import re
import shutil
import tempfile
import urlparse

import pyauto_functional  # Must be imported before pyauto
import pyauto
import test_utils

class OmniboxTest(pyauto.PyUITest):
  """TestCase for Omnibox."""

  def Debug(self):
    """Test method for experimentation.

    This method will not run automatically.
    """
    import pprint
    import time
    pp = pprint.PrettyPrinter(indent=2)
    while True:
      pp.pprint(self.GetOmniboxInfo().omniboxdict)
      time.sleep(1)

  def testFocusOnStartup(self):
    """Verify that omnibox has focus on startup."""
    self.WaitUntilOmniboxReadyHack()
    self.assertTrue(self.GetOmniboxInfo().Properties('has_focus'))

  def testHistoryResult(self):
    """Verify that omnibox can fetch items from history."""
    url = self.GetFileURLForDataPath('title2.html')
    title = 'Title Of Awesomeness'
    self.AppendTab(pyauto.GURL(url))
    def _VerifyHistoryResult(query_list, description, windex=0):
      """Verify result matching given description for given list of queries."""
      for query_text in query_list:
        matches = test_utils.GetOmniboxMatchesFor(self,
            query_text, windex=windex, attr_dict={'description': description})
        self.assertTrue(matches)
        self.assertEqual(1, len(matches))
        item = matches[0]
        self.assertEqual(url, item['destination_url'])
    # Query using URL & title
    _VerifyHistoryResult([url, title], title)
    # Verify results in another tab
    self.AppendTab(pyauto.GURL())
    _VerifyHistoryResult([url, title], title)
    # Verify results in another window
    self.OpenNewBrowserWindow(True)
    self.WaitUntilOmniboxReadyHack(windex=1)
    _VerifyHistoryResult([url, title], title, windex=1)
    # Verify results in an incognito window
    self.RunCommand(pyauto.IDC_NEW_INCOGNITO_WINDOW)
    self.WaitUntilOmniboxReadyHack(windex=2)
    _VerifyHistoryResult([url, title], title, windex=2)

  def _VerifyOmniboxURLMatches(self, url, description, windex=0):
    """Verify URL match results from the Omnibox.

    Args:
      url: the url to use
      description: the string description within history page and google search
                   to match against
      windex: the window index to work on. Defaults to 0 (first window)
    """
    matches_description = test_utils.GetOmniboxMatchesFor(self,
        url, windex=windex, attr_dict={'description': description})
    self.assertEqual(1, len(matches_description))
    if description == 'Google Search':
      self.assertTrue(re.match('http://www.google.com/search.+',
                               matches_description[0]['destination_url']))
    else:
      self.assertEqual(url, matches_description[0]['destination_url'])

  def testFetchHistoryResultItems(self):
    """Verify omnibox fetches history items in second tab, win and Incognito."""
    url = self.GetFileURLForDataPath('title2.html')
    title = 'Title Of Awesomeness'
    desc = 'Google Search'
    # fetch history page item in the second tab.
    self.AppendTab(pyauto.GURL(url))
    self._VerifyOmniboxURLMatches(url, title)
    # fetch history page items in the second window.
    self.OpenNewBrowserWindow(True)
    self.NavigateToURL(url, 1, 0)
    self._VerifyOmniboxURLMatches(url, title, windex=1)
    # fetch google search items in Incognito window.
    self.RunCommand(pyauto.IDC_NEW_INCOGNITO_WINDOW)
    self.NavigateToURL(url, 2, 0)
    self._VerifyOmniboxURLMatches(url, desc, windex=2)

  def testSelect(self):
    """Verify omnibox popup selection."""
    url1 = self.GetFileURLForDataPath('title2.html')
    url2 = self.GetFileURLForDataPath('title1.html')
    title1 = 'Title Of Awesomeness'
    self.NavigateToURL(url1)
    self.NavigateToURL(url2)
    matches = test_utils.GetOmniboxMatchesFor(self, 'file://')
    self.assertTrue(matches)
    # Find the index of match for url1
    index = None
    for i, match in enumerate(matches):
      if match['description'] == title1:
        index = i
    self.assertTrue(index is not None)
    self.OmniboxMovePopupSelection(index)  # Select url1 line in popup
    self.assertEqual(url1, self.GetOmniboxInfo().Text())
    self.OmniboxAcceptInput()
    self.assertEqual(title1, self.GetActiveTabTitle())

  def testGoogleSearch(self):
    """Verify Google search item in omnibox results."""
    search_text = 'hello world'
    verify_str = 'Google Search'
    url_re = 'http://www.google.com/search\?.*q=hello\+world.*'
    matches_description = test_utils.GetOmniboxMatchesFor(self,
        search_text, attr_dict={'description': verify_str})
    self.assertTrue(matches_description)
    # There should be a least one entry with the description Google. Suggest
    # results may end up having 'Google Search' in them, so use >=.
    self.assertTrue(len(matches_description) >= 1)
    item = matches_description[0]
    self.assertTrue(re.search(url_re, item['destination_url']))
    self.assertEqual('search-what-you-typed', item['type'])

  def testInlinAutoComplete(self):
    """Verify inline autocomplete for a pre-visited url."""
    self.NavigateToURL('http://www.google.com')
    matches = test_utils.GetOmniboxMatchesFor(self, 'goog')
    self.assertTrue(matches)
    # Omnibox should suggest auto completed url as the first item
    matches_description = matches[0]
    self.assertTrue('www.google.com' in matches_description['contents'])
    self.assertEqual('history-url', matches_description['type'])
    # The url should be inline-autocompleted in the omnibox
    self.assertTrue('google.com' in self.GetOmniboxInfo().Text())

  def testCrazyFilenames(self):
    """Test omnibox query with filenames containing special chars.

    The files are created on the fly and cleaned after use.
    """
    filename = os.path.join(self.DataDir(), 'downloads', 'crazy_filenames.txt')
    zip_names = self.EvalDataFrom(filename)
    # We got .zip filenames. Change them to .html
    crazy_filenames = [x.replace('.zip', '.html') for x in zip_names]
    title = 'given title'

    def _CreateFile(name):
      """Create the given html file."""
      fp = open(name, 'w')  # name could be unicode
      print >>fp, '<title>%s</title>' % title
      print >>fp, 'This is a junk file named
%s
' % repr(name)
      print >>fp, ''
      fp.close()

    crazy_fileurls = []
    # Temp dir for hosting crazy filenames.
    temp_dir = tempfile.mkdtemp(prefix='omnibox')
    # Windows has a dual nature dealing with unicode filenames.
    # While the files are internally saved as unicode, there's a non-unicode
    # aware API that returns a locale-dependent coding on the true unicode
    # filenames.  This messes up things.
    # Filesystem-interfacing functions like os.listdir() need to
    # be given unicode strings to "do the right thing" on win.
    # Ref: http://boodebr.org/main/python/all-about-python-and-unicode
    try:
      for filename in crazy_filenames:  # filename is unicode.
        file_path = os.path.join(temp_dir, filename.encode('utf-8'))
        _CreateFile(os.path.join(temp_dir, filename))
        file_url = self.GetFileURLForPath(file_path)
        crazy_fileurls.append(file_url)
        self.NavigateToURL(file_url)

      # Verify omnibox queries.
      for file_url in crazy_fileurls:
        matches = test_utils.GetOmniboxMatchesFor(self,
            file_url, attr_dict={'type': 'url-what-you-typed',
                                 'description': title})
        self.assertTrue(matches)
        self.assertEqual(1, len(matches))
        self.assertTrue(os.path.basename(file_url) in
                        matches[0]['destination_url'])
    finally:
      shutil.rmtree(unicode(temp_dir))  # unicode so that win treats nicely.

  def testSuggest(self):
    """Verify suggested results in omnibox."""
    matches = test_utils.GetOmniboxMatchesFor(self, 'apple')
    self.assertTrue(matches)
    self.assertTrue([x for x in matches if x['type'] == 'search-suggest'])

  def testDifferentTypesOfResults(self):
    """Verify different types of results from omnibox.

    This includes history result, bookmark result, suggest results.
    """
    url = 'http://www.google.com/'
    title = 'Google'
    search_string = 'google'
    self.AddBookmarkURL(  # Add a bookmark
        self.GetBookmarkModel().BookmarkBar()['id'], 0, title, url)
    self.NavigateToURL(url)  # Build up history
    matches = test_utils.GetOmniboxMatchesFor(self, search_string)
    self.assertTrue(matches)
    # Verify starred result (indicating bookmarked url)
    self.assertTrue([x for x in matches if x['starred'] == True])
    for item_type in ('history-url', 'search-what-you-typed',
                      'search-suggest',):
      self.assertTrue([x for x in matches if x['type'] == item_type])

  def testSuggestPref(self):
    """Verify no suggests for omnibox when suggested-services disabled."""
    search_string = 'apple'
    self.assertTrue(self.GetPrefsInfo().Prefs(pyauto.kSearchSuggestEnabled))
    matches = test_utils.GetOmniboxMatchesFor(self, search_string)
    self.assertTrue(matches)
    self.assertTrue([x for x in matches if x['type'] == 'search-suggest'])
    # Disable suggest-service
    self.SetPrefs(pyauto.kSearchSuggestEnabled, False)
    self.assertFalse(self.GetPrefsInfo().Prefs(pyauto.kSearchSuggestEnabled))
    matches = test_utils.GetOmniboxMatchesFor(self, search_string)
    self.assertTrue(matches)
    # Verify there are no suggest results
    self.assertFalse([x for x in matches if x['type'] == 'search-suggest'])

  def testAutoCompleteForSearch(self):
    """Verify omnibox autocomplete for search."""
    search_string = 'youtu'
    verify_string = 'youtube'
    matches = test_utils.GetOmniboxMatchesFor(self, search_string)
    # retrieve last contents element.
    matches_description = matches[-1]['contents'].split()
    self.assertEqual(verify_string, matches_description[0])

  def _CheckBookmarkResultForVariousInputs(self, url, title, windex=0):
    """Check if we get the Bookmark for complete and partial inputs."""
    # Check if the complete URL would get the bookmark.
    url_matches = test_utils.GetOmniboxMatchesFor(self, url, windex=windex)
    self._VerifyHasBookmarkResult(url_matches)
    # Check if the complete title would get the bookmark.
    title_matches = test_utils.GetOmniboxMatchesFor(self, title, windex=windex)
    self._VerifyHasBookmarkResult(title_matches)
    # Check if the partial URL would get the bookmark.
    split_url = urlparse.urlsplit(url)
    partial_url = test_utils.GetOmniboxMatchesFor(self,
        split_url.scheme, windex=windex)
    self._VerifyHasBookmarkResult(partial_url)
    # Check if the partial title would get the bookmark.
    split_title = title.split()
    search_term = split_title[len(split_title) - 1]
    partial_title = test_utils.GetOmniboxMatchesFor(self,
        search_term, windex=windex)
    self._VerifyHasBookmarkResult(partial_title)

  def _GotContentHistory(self, search_text, url):
    """Determines if omnibox returns a previously visited page for given
       search text
    """
    # Omnibox doesn't change results if searching the same text repeatedly.
    # So setting '' in omnibox before the next repeated search.
    self.SetOmniboxText('')
    matches = test_utils.GetOmniboxMatchesFor(self, search_text)
    matches_description = [x for x in matches if x['destination_url'] == url]
    return 1 == len(matches_description)

  def testContentHistory(self):
    """Verify omnibox results when entering page content

    Test verifies that visited page shows up in omnibox on entering page
    content.
    """
    url = self.GetFileURLForPath(
        os.path.join(self.DataDir(), 'find_in_page', 'largepage.html'))
    self.NavigateToURL(url)
    self.assertTrue(self.WaitUntil(
        lambda: self._GotContentHistory('British throne', url)))

  def testOmniboxSearchHistory(self):
    """Verify Page navigation/search from omnibox are added to the history."""
    url = self.GetFileURLForDataPath('title2.html')
    self.NavigateToURL(url)
    self.AppendTab(pyauto.GURL('about:blank'))
    self.SetOmniboxText('java')
    self.WaitUntilOmniboxQueryDone()
    self.OmniboxAcceptInput()
    history = self.GetHistoryInfo().History()
    self.assertEqual(2, len(history))
    self.assertEqual(url, history[1]['url'])
    self.assertEqual('java - Google Search', history[0]['title'])

  def _VerifyHasBookmarkResult(self, matches):
    """Verify that we have a bookmark result."""
    matches_starred = [result for result in matches if result['starred']]
    self.assertTrue(matches_starred)
    self.assertEqual(1, len(matches_starred))

  def testBookmarkResultInNewTabAndWindow(self):
    """Verify that omnibox can recognize a bookmark within search options
    in new tabs and windows."""
    url = self.GetFileURLForDataPath('title2.html')
    self.NavigateToURL(url)
    title = 'This is Awesomeness'
    bookmarks = self.GetBookmarkModel()
    bar_id = bookmarks.BookmarkBar()['id']
    self.AddBookmarkURL(bar_id, 0, title, url)
    bookmarks = self.GetBookmarkModel()
    nodes = bookmarks.FindByTitle(title)
    self.AppendTab(pyauto.GURL(url))
    self._CheckBookmarkResultForVariousInputs(url, title)
    self.OpenNewBrowserWindow(True)
    self.assertEqual(2, self.GetBrowserWindowCount())
    self.NavigateToURL(url, 1, 0)
    self._CheckBookmarkResultForVariousInputs(url, title, windex=1)
    self.RunCommand(pyauto.IDC_NEW_INCOGNITO_WINDOW)
    self.assertEqual(3, self.GetBrowserWindowCount())
    self.NavigateToURL(url, 2, 0)
    self._CheckBookmarkResultForVariousInputs(url, title, windex=2)

  def testAutoCompleteForNonAsciiSearch(self):
    """Verify that able to search and autocomplete with non-ASCII incomplete
    search keyword."""
    search_string = u'\u767e'
    verify_string = u'\u767e\u5ea6\u4e00\u4e0b'
    matches = test_utils.GetOmniboxMatchesFor(self, search_string)
    self.assertTrue(verify_string in matches[-1]['contents'])

  def _InstallAndVerifySamplePackagedApp(self):
    """Installs a sample packaged app and verifies the install is successful.

    Returns:
      The string ID of the installed app.
    """
    app_crx_file = os.path.abspath(os.path.join(self.DataDir(),
                                   'pyauto_private', 'apps', 'countdown.crx'))
    installed_app_id = self.InstallApp(app_crx_file)
    self.assertTrue(installed_app_id, msg='App install failed.')

  def testAppSearch(self):
    """Verify that we can search for installed apps"""
    self._InstallAndVerifySamplePackagedApp()
    self.SetOmniboxText('countdown')
    self.WaitUntilOmniboxQueryDone()
    self.assertTrue('countdown' in self.GetOmniboxInfo().Text())
    self.OmniboxAcceptInput()
    app_url = 'chrome-extension:' \
              '//aeabikdlfbfeihglecobdkdflahfgcpd/launchLocalPath.html'
    self.assertEqual(app_url, self.GetActiveTabURL().spec())

if __name__ == '__main__':
  pyauto_functional.Main()
<|endoftext|>


<commit_before>from xml.dom import NamespaceErr
import hashlib
from urllib.parse import urlparse
import re
from defusedxml import ElementTree as ET
from dojo.models import Endpoint, Finding

__author__ = 'dr3dd589'

class SslscanXMLParser(object):
    def __init__(self, file, test):
        self.dupes = dict()
        self.items = ()
        if file is None:
            return

        tree = ET.parse(file)
        # get root of tree.
        root = tree.getroot()
        if 'Sessions' not in root.tag:
            raise NamespaceErr("This doesn't seem to be a valid Webinspect xml file.")<commit_msg>sslscan partially done<commit_after>from xml.dom import NamespaceErr
import hashlib
from urllib.parse import urlparse
import re
from defusedxml import ElementTree as ET
from dojo.models import Endpoint, Finding

__author__ = 'dr3dd589'

class SslscanXMLParser(object):
    def __init__(self, file, test):
        self.dupes = dict()
        self.items = ()
        if file is None:
            return

        tree = ET.parse(file)
        # get root of tree.
        root = tree.getroot()
        if 'document' not in root.tag:
            raise NamespaceErr("This doesn't seem to be a valid sslscan xml file.")

        for ssltest in root:
            for target in ssltest:
                title = ""
                severity = ""
                description = ""
                severity = "Info"
                url = ssltest.attrib['host']
                port = ssltest.attrib['port']
                parsedUrl = urlparse(url)
                protocol = parsedUrl.scheme
                query = parsedUrl.query
                fragment = parsedUrl.fragment
                path = parsedUrl.path
                try:
                    (host, port) = parsedUrl.netloc.split(':')
                except:
                    host = parsedUrl.netloc
                if target.tag == "heartbleed" and target.attrib['vulnerable'] == '1':
                    title = "heartbleed" + " | " + target.attrib['sslversion']
                    description = "**heartbleed** :" + "\n\n" + \
                                "**sslversion** : " + target.attrib['sslversion'] + "\n"
                if target.tag == "cipher" and target.attrib['strength'] not in ['acceptable', 'strong']:
                    title = "cipher" + " | "+ target.attrib['sslversion']
                    description = "**Cipher** : " + target.attrib['cipher'] + "\n\n" + \
                                "**Status** : " + target.attrib['status'] + "\n\n" + \
                                "**strength** : " + target.attrib['strength'] + "\n\n" + \
                                "**sslversion** : " + target.attrib['sslversion'] + "\n"

                if title and description is not None:
                    dupe_key = hashlib.md5(str(description + title).encode('utf-8')).hexdigest()
                    if dupe_key in self.dupes:
                        finding = self.dupes[dupe_key]
                        if finding.references:
                            finding.references = finding.references
                        self.dupes[dupe_key] = finding
                    else:
                        self.dupes[dupe_key] = True

                        finding = Finding(
                            title=title,
                            test=test,
                            active=False,
                            verified=False,
                            description=description,
                            severity=severity,
                            numerical_severity=Finding.get_numerical_severity(severity),
                            dynamic_finding=True,)
                        finding.unsaved_endpoints = list()
                        self.dupes[dupe_key] = finding

                        if url is not None:
                            finding.unsaved_endpoints.append(Endpoint(
                                host=host,
                                port=port,
                                path=path,
                                protocol=protocol,
                                query=query,
                                fragment=fragment,))
                self.items = self.dupes.values()

<|endoftext|>