<commit_before># coding: utf-8
from __future__ import unicode_literals

import re
import json
import pickle
import textwrap
import unittest
import importlib
import importlib_metadata
import pyfakefs.fake_filesystem_unittest as ffs

from . import fixtures
from .. import (
    Distribution, EntryPoint, MetadataPathFinder,
    PackageNotFoundError, distributions,
    entry_points, metadata, version,
    )

try:
    from builtins import str as text
except ImportError:
    from __builtin__ import unicode as text

class BasicTests(fixtures.DistInfoPkg, unittest.TestCase):
    version_pattern = r'\d+\.\d+(\.\d)?'

    def test_retrieves_version_of_self(self):
        dist = Distribution.from_name('distinfo-pkg')
        assert isinstance(dist.version, text)
        assert re.match(self.version_pattern, dist.version)

    def test_for_name_does_not_exist(self):
        with self.assertRaises(PackageNotFoundError):
            Distribution.from_name('does-not-exist')

    def test_new_style_classes(self):
        self.assertIsInstance(Distribution, type)
        self.assertIsInstance(MetadataPathFinder, type)

class ImportTests(fixtures.DistInfoPkg, unittest.TestCase):
    def test_import_nonexistent_module(self):
        # Ensure that the MetadataPathFinder does not crash an import of a
        # non-existent module.
        with self.assertRaises(ImportError):
            importlib.import_module('does_not_exist')

    def test_resolve(self):
        entries = dict(entry_points()['entries'])
        ep = entries['main']
        self.assertEqual(ep.load().__name__, "main")

    def test_entrypoint_with_colon_in_name(self):
        entries = dict(entry_points()['entries'])
        ep = entries['ns:sub']
        self.assertEqual(ep.value, 'mod:main')

    def test_resolve_without_attr(self):
        ep = EntryPoint(
            name='ep',
            value='importlib_metadata',
            group='grp',
            )
        assert ep.load() is importlib_metadata

class NameNormalizationTests(
        fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
    @staticmethod
    def pkg_with_dashes(site_dir):
        """
        Create minimal metadata for a package with dashes
        in the name (and thus underscores in the filename).
        """
        metadata_dir = site_dir / 'my_pkg.dist-info'
        metadata_dir.mkdir()
        metadata = metadata_dir / 'METADATA'
        with metadata.open('w') as strm:
            strm.write('Version: 1.0\n')
        return 'my-pkg'

    def test_dashes_in_dist_name_found_as_underscores(self):
        """
        For a package with a dash in the name, the dist-info metadata
        uses underscores in the name. Ensure the metadata loads.
        """
        pkg_name = self.pkg_with_dashes(self.site_dir)
        assert version(pkg_name) == '1.0'

    @staticmethod
    def pkg_with_mixed_case(site_dir):
        """
        Create minimal metadata for a package with mixed case
        in the name.
        """
        metadata_dir = site_dir / 'CherryPy.dist-info'
        metadata_dir.mkdir()
        metadata = metadata_dir / 'METADATA'
        with metadata.open('w') as strm:
            strm.write('Version: 1.0\n')
        return 'CherryPy'

    def test_dist_name_found_as_any_case(self):
        """
        Ensure the metadata loads when queried with any case.
        """
        pkg_name = self.pkg_with_mixed_case(self.site_dir)
        assert version(pkg_name) == '1.0'
        assert version(pkg_name.lower()) == '1.0'
        assert version(pkg_name.upper()) == '1.0'

class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
    @staticmethod
    def pkg_with_non_ascii_description(site_dir):
        """
        Create minimal metadata for a package with non-ASCII in
        the description.
        """
        metadata_dir = site_dir / 'portend.dist-info'
        metadata_dir.mkdir()
        metadata = metadata_dir / 'METADATA'
        with metadata.open('w', encoding='utf-8') as fp:
            fp.write('Description: pôrˈtend\n')
        return 'portend'

    @staticmethod
    def pkg_with_non_ascii_description_egg_info(site_dir):
        """
        Create minimal metadata for an egg-info package with
        non-ASCII in the description.
        """
        metadata_dir = site_dir / 'portend.dist-info'
        metadata_dir.mkdir()
        metadata = metadata_dir / 'METADATA'
        with metadata.open('w', encoding='utf-8') as fp:
            fp.write(textwrap.dedent("""
                Name: portend

                pôrˈtend
                """).lstrip())
        return 'portend'

    def test_metadata_loads(self):
        pkg_name = self.pkg_with_non_ascii_description(self.site_dir)
        meta = metadata(pkg_name)
        assert meta['Description'] == 'pôrˈtend'

    def test_metadata_loads_egg_info(self):
        pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir)
        meta = metadata(pkg_name)
        assert meta.get_payload() == 'pôrˈtend\n'

class DiscoveryTests(fixtures.EggInfoPkg,
                     fixtures.DistInfoPkg,
                     unittest.TestCase):

    def test_package_discovery(self):
        dists = list(distributions())
        assert all(
            isinstance(dist, Distribution)
            for dist in dists
            )
        assert any(
            dist.metadata['Name'] == 'egginfo-pkg'
            for dist in dists
            )
        assert any(
            dist.metadata['Name'] == 'distinfo-pkg'
            for dist in dists
            )

    def test_invalid_usage(self):
        with self.assertRaises(ValueError):
            list(distributions(context='something', name='else'))

class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
    def test_egg_info(self):
        # make an `EGG-INFO` directory that's unrelated
        self.site_dir.joinpath('EGG-INFO').mkdir()
        # used to crash with `IsADirectoryError`
        with self.assertRaises(PackageNotFoundError):
            version('unknown-package')

    def test_egg(self):
        egg = self.site_dir.joinpath('foo-3.6.egg')
        egg.mkdir()
        with self.add_sys_path(egg):
            with self.assertRaises(PackageNotFoundError):
                version('foo')

class MissingSysPath(fixtures.OnSysPath, unittest.TestCase):
    site_dir = '/does-not-exist'

    def test_discovery(self):
        """
        Discovering distributions should succeed even if
        there is an invalid path on sys.path.
        """
        importlib_metadata.distributions()

class InaccessibleSysPath(fixtures.OnSysPath, ffs.TestCase):
    site_dir = '/access-denied'

    def setUp(self):
        super(InaccessibleSysPath, self).setUp()
        self.setUpPyfakefs()
        self.fs.create_dir(self.site_dir, perm_bits=000)

    def test_discovery(self):
        """
        Discovering distributions should succeed even if
        there is an invalid path on sys.path.
        """
        list(importlib_metadata.distributions())

class TestEntryPoints(unittest.TestCase):
    def __init__(self, *args):
        super(TestEntryPoints, self).__init__(*args)
        self.ep = importlib_metadata.EntryPoint('name', 'value', 'group')

    def test_entry_point_pickleable(self):
        revived = pickle.loads(pickle.dumps(self.ep))
        assert revived == self.ep

    def test_immutable(self):
        """EntryPoints should be immutable"""
        with self.assertRaises(AttributeError):
            self.ep.name = 'badactor'

    def test_repr(self):
        assert 'EntryPoint' in repr(self.ep)
        assert 'name=' in repr(self.ep)
        assert "'name'" in repr(self.ep)

    def test_hashable(self):
        """EntryPoints should be hashable"""
        hash(self.ep)

    def test_json_dump(self):
        """
        json should not expect to be able to dump an EntryPoint
        """
        with self.assertRaises(Exception):
            json.dumps(self.ep)

    def test_module(self):
        assert self.ep.module == 'value'

    def test_attr(self):
        assert self.ep.attr is None

class FileSystem(
        fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder,
        unittest.TestCase):
    def test_unicode_dir_on_sys_path(self):
        """
        Ensure a Unicode subdirectory of a directory on sys.path
        does not crash.
        """
        fixtures.build_files(
            {self.unicode_filename(): {}},
            prefix=self.site_dir,
            )
        list(distributions())
<commit_msg>Add test capturing expectation that users should receive a hint as to the cause of a PackageNotFoundError if the package metadata is missing. Ref #124.<commit_after># coding: utf-8
from __future__ import unicode_literals

import re
import json
import pickle
import textwrap
import unittest
import importlib
import importlib_metadata
import pyfakefs.fake_filesystem_unittest as ffs

from . import fixtures
from .. import (
    Distribution, EntryPoint, MetadataPathFinder,
    PackageNotFoundError, distributions,
    entry_points, metadata, version,
    )

try:
    from builtins import str as text
except ImportError:
    from __builtin__ import unicode as text

class BasicTests(fixtures.DistInfoPkg, unittest.TestCase):
    version_pattern = r'\d+\.\d+(\.\d)?'

    def test_retrieves_version_of_self(self):
        dist = Distribution.from_name('distinfo-pkg')
        assert isinstance(dist.version, text)
        assert re.match(self.version_pattern, dist.version)

    def test_for_name_does_not_exist(self):
        with self.assertRaises(PackageNotFoundError):
            Distribution.from_name('does-not-exist')

    def test_package_not_found_mentions_metadata(self):
        """
        When a package is not found, that could indicate that the
        packgae is not installed or that it is installed without
        metadata. Ensure the exception mentions metadata to help
        guide users toward the cause. See #124.
        """
        with self.assertRaises(PackageNotFoundError) as ctx:
            Distribution.from_name('does-not-exist')

        assert "metadata" in str(ctx.exception)

    def test_new_style_classes(self):
        self.assertIsInstance(Distribution, type)
        self.assertIsInstance(MetadataPathFinder, type)

class ImportTests(fixtures.DistInfoPkg, unittest.TestCase):
    def test_import_nonexistent_module(self):
        # Ensure that the MetadataPathFinder does not crash an import of a
        # non-existent module.
        with self.assertRaises(ImportError):
            importlib.import_module('does_not_exist')

    def test_resolve(self):
        entries = dict(entry_points()['entries'])
        ep = entries['main']
        self.assertEqual(ep.load().__name__, "main")

    def test_entrypoint_with_colon_in_name(self):
        entries = dict(entry_points()['entries'])
        ep = entries['ns:sub']
        self.assertEqual(ep.value, 'mod:main')

    def test_resolve_without_attr(self):
        ep = EntryPoint(
            name='ep',
            value='importlib_metadata',
            group='grp',
            )
        assert ep.load() is importlib_metadata

class NameNormalizationTests(
        fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
    @staticmethod
    def pkg_with_dashes(site_dir):
        """
        Create minimal metadata for a package with dashes
        in the name (and thus underscores in the filename).
        """
        metadata_dir = site_dir / 'my_pkg.dist-info'
        metadata_dir.mkdir()
        metadata = metadata_dir / 'METADATA'
        with metadata.open('w') as strm:
            strm.write('Version: 1.0\n')
        return 'my-pkg'

    def test_dashes_in_dist_name_found_as_underscores(self):
        """
        For a package with a dash in the name, the dist-info metadata
        uses underscores in the name. Ensure the metadata loads.
        """
        pkg_name = self.pkg_with_dashes(self.site_dir)
        assert version(pkg_name) == '1.0'

    @staticmethod
    def pkg_with_mixed_case(site_dir):
        """
        Create minimal metadata for a package with mixed case
        in the name.
        """
        metadata_dir = site_dir / 'CherryPy.dist-info'
        metadata_dir.mkdir()
        metadata = metadata_dir / 'METADATA'
        with metadata.open('w') as strm:
            strm.write('Version: 1.0\n')
        return 'CherryPy'

    def test_dist_name_found_as_any_case(self):
        """
        Ensure the metadata loads when queried with any case.
        """
        pkg_name = self.pkg_with_mixed_case(self.site_dir)
        assert version(pkg_name) == '1.0'
        assert version(pkg_name.lower()) == '1.0'
        assert version(pkg_name.upper()) == '1.0'

class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
    @staticmethod
    def pkg_with_non_ascii_description(site_dir):
        """
        Create minimal metadata for a package with non-ASCII in
        the description.
        """
        metadata_dir = site_dir / 'portend.dist-info'
        metadata_dir.mkdir()
        metadata = metadata_dir / 'METADATA'
        with metadata.open('w', encoding='utf-8') as fp:
            fp.write('Description: pôrˈtend\n')
        return 'portend'

    @staticmethod
    def pkg_with_non_ascii_description_egg_info(site_dir):
        """
        Create minimal metadata for an egg-info package with
        non-ASCII in the description.
        """
        metadata_dir = site_dir / 'portend.dist-info'
        metadata_dir.mkdir()
        metadata = metadata_dir / 'METADATA'
        with metadata.open('w', encoding='utf-8') as fp:
            fp.write(textwrap.dedent("""
                Name: portend

                pôrˈtend
                """).lstrip())
        return 'portend'

    def test_metadata_loads(self):
        pkg_name = self.pkg_with_non_ascii_description(self.site_dir)
        meta = metadata(pkg_name)
        assert meta['Description'] == 'pôrˈtend'

    def test_metadata_loads_egg_info(self):
        pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir)
        meta = metadata(pkg_name)
        assert meta.get_payload() == 'pôrˈtend\n'

class DiscoveryTests(fixtures.EggInfoPkg,
                     fixtures.DistInfoPkg,
                     unittest.TestCase):

    def test_package_discovery(self):
        dists = list(distributions())
        assert all(
            isinstance(dist, Distribution)
            for dist in dists
            )
        assert any(
            dist.metadata['Name'] == 'egginfo-pkg'
            for dist in dists
            )
        assert any(
            dist.metadata['Name'] == 'distinfo-pkg'
            for dist in dists
            )

    def test_invalid_usage(self):
        with self.assertRaises(ValueError):
            list(distributions(context='something', name='else'))

class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
    def test_egg_info(self):
        # make an `EGG-INFO` directory that's unrelated
        self.site_dir.joinpath('EGG-INFO').mkdir()
        # used to crash with `IsADirectoryError`
        with self.assertRaises(PackageNotFoundError):
            version('unknown-package')

    def test_egg(self):
        egg = self.site_dir.joinpath('foo-3.6.egg')
        egg.mkdir()
        with self.add_sys_path(egg):
            with self.assertRaises(PackageNotFoundError):
                version('foo')

class MissingSysPath(fixtures.OnSysPath, unittest.TestCase):
    site_dir = '/does-not-exist'

    def test_discovery(self):
        """
        Discovering distributions should succeed even if
        there is an invalid path on sys.path.
        """
        importlib_metadata.distributions()

class InaccessibleSysPath(fixtures.OnSysPath, ffs.TestCase):
    site_dir = '/access-denied'

    def setUp(self):
        super(InaccessibleSysPath, self).setUp()
        self.setUpPyfakefs()
        self.fs.create_dir(self.site_dir, perm_bits=000)

    def test_discovery(self):
        """
        Discovering distributions should succeed even if
        there is an invalid path on sys.path.
        """
        list(importlib_metadata.distributions())

class TestEntryPoints(unittest.TestCase):
    def __init__(self, *args):
        super(TestEntryPoints, self).__init__(*args)
        self.ep = importlib_metadata.EntryPoint('name', 'value', 'group')

    def test_entry_point_pickleable(self):
        revived = pickle.loads(pickle.dumps(self.ep))
        assert revived == self.ep

    def test_immutable(self):
        """EntryPoints should be immutable"""
        with self.assertRaises(AttributeError):
            self.ep.name = 'badactor'

    def test_repr(self):
        assert 'EntryPoint' in repr(self.ep)
        assert 'name=' in repr(self.ep)
        assert "'name'" in repr(self.ep)

    def test_hashable(self):
        """EntryPoints should be hashable"""
        hash(self.ep)

    def test_json_dump(self):
        """
        json should not expect to be able to dump an EntryPoint
        """
        with self.assertRaises(Exception):
            json.dumps(self.ep)

    def test_module(self):
        assert self.ep.module == 'value'

    def test_attr(self):
        assert self.ep.attr is None

class FileSystem(
        fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder,
        unittest.TestCase):
    def test_unicode_dir_on_sys_path(self):
        """
        Ensure a Unicode subdirectory of a directory on sys.path
        does not crash.
        """
        fixtures.build_files(
            {self.unicode_filename(): {}},
            prefix=self.site_dir,
            )
        list(distributions())
<|endoftext|>


<commit_before>import urllib2
import logging

from django.conf.urls import url
from django.contrib.sites.models import Site
from django.core.urlresolvers import resolve
from django.http import Http404
from django.shortcuts import render_to_response
from django.template.context import RequestContext
import requests
from allauth.socialaccount.models import SocialToken
from core.utils.url import wrap_full

from dss import settings
from spa.models import Playlist
from spa.models.mix import Mix
from spa.models.userprofile import UserProfile

logger = logging.getLogger(__name__)

"""
    Handles callbacks from facebook and twitter
"""

def _getPayload(request):
    return {
        "app_id": settings.FACEBOOK_APP_ID,
        "site_url": 'http://%s' % Site.objects.get_current().domain,
        "site_image_url": '%s/img/dss-large.png' % settings.STATIC_URL,
    }

def mix(request, args):
    try:
        if 'mix_id' in args:
            mix = Mix.objects.get(pk=args['mix_id'])
        else:
            mix = Mix.objects.get(slug=args['slug'])
    except Mix.DoesNotExist:
        raise Http404

    image = mix.get_image_url('400x400')
    audio_url = mix.get_stream_path()
    mix_url = mix.get_absolute_url()
    default = _getPayload(request)
    extras = {
        "description": mix.description.replace('
', '\n'),
        "title": mix.title,
        "image_url": image,
        "audio_url": audio_url,
        "mix_url": 'http://%s%s' % (Site.objects.get_current().domain, mix_url)
    }
    payload = dict(default.items() + extras.items())
    response = render_to_response(
        'inc/facebook/mix.html',
        payload,
        context_instance=RequestContext(request)
    )
    return response

def playlist(request, args):
    try:
        playlist = Playlist.objects.get(slug=args['slug'])
    except Playlist.DoesNotExist:
        raise Http404

    site_url = settings.DEBUG_URL if settings.DEBUG else Site.objects.get_current().domain
    image = playlist.get_image_url('400x400')
    playlist_url = "%s%s" % (site_url, playlist.get_absolute_url())
    default = _getPayload(request)
    extras = {
        "description": "Deep South Sounds Playlist by %s" % playlist.user.get_nice_name(),
        "title": playlist.name,
        "image_url": image,
        "playlist_url": playlist_url
    }
    payload = dict(default.items() + extras.items())
    response = render_to_response(
        'inc/facebook/playlist.html',
        payload,
        context_instance=RequestContext(request)
    )
    print response.content
    return response

def user(request, args):
    try:
        user = UserProfile.objects.get(slug=args['user_id'])
    except UserProfile.DoesNotExist:
        raise Http404

    image = user.get_avatar_image()
    profile_url = user.get_profile_url()
    default = _getPayload(request)
    extras = {
        "title": user.get_nice_name(),
        "description": user.get_profile_description().replace('
', '\n'),
        "profile_url": wrap_full(profile_url),
        "image_url": image,
    }
    payload = dict(default.items() + extras.items())
    response = render_to_response(
        'inc/facebook/user.html',
        payload,
        context_instance=RequestContext(request)
    )
    return response

def index(request):
    response = render_to_response(
        "inc/facebook/index.html",
        _getPayload(request),
        context_instance=RequestContext(request))
    return response

def social_redirect(request):
    try:
        resolver = resolve('/social' + request.path)
        if resolver is not None:
            logger.debug("Resolver successfully resolved")
            return resolver.func(request, resolver.kwargs)
        else:
            logger.debug("No resolver found for: $%s" % request.path)
    except Http404:
        logger.debug("404 on resolver: $%s" % request.path)
        return index(request)
    except Exception, ex:
        logger.debug("Unhandled exception in social_redirect: $%s" % ex)
        return index(request)

def post_like(request, mix):
    try:
        tokens = SocialToken.objects.filter(
            account__user=request.user,
            account__provider='facebook')
        for token in tokens:
            url = 'https://graph.facebook.com/%s/og.likes' % token.account.uid
            values = {
                'access_token': token.token,
                'object': mix.get_full_url(),
            }
            response = requests.post(url, data=values)
            if response.status_code == 200:
                print "Returned %s" % response.json
                return response.json['id']
            else:
                print "Returned status code of %s" % response.status_code
    except urllib2.HTTPError, httpEx:
        print httpEx.message
    except Exception, ex:
        print ex.message
    return ""

def delete_like(request, uid):
    try:
        tokens = SocialToken.objects.filter(account__user=request.user, account__provider='facebook')
        for token in tokens:
            url = "https://graph.facebook.com/%s" % uid
            values = {
                'access_token': token.token,
            }
            response = requests.delete(url, data=values)
            return response
    except Exception, ex:
        print "Error talking with facebook: %s" % ex.message

<commit_msg>Fixed site url issue in FB playlist redirect<commit_after>import urllib2
import logging

from django.conf.urls import url
from django.contrib.sites.models import Site
from django.core.urlresolvers import resolve
from django.http import Http404
from django.shortcuts import render_to_response
from django.template.context import RequestContext
import requests
from allauth.socialaccount.models import SocialToken
from core.utils.url import wrap_full

from dss import settings
from spa.models import Playlist
from spa.models.mix import Mix
from spa.models.userprofile import UserProfile

logger = logging.getLogger(__name__)

"""
    Handles callbacks from facebook and twitter
"""

def _getPayload(request):
    return {
        "app_id": settings.FACEBOOK_APP_ID,
        "site_url": 'http://%s' % Site.objects.get_current().domain,
        "site_image_url": '%s/img/dss-large.png' % settings.STATIC_URL,
    }

def mix(request, args):
    try:
        if 'mix_id' in args:
            mix = Mix.objects.get(pk=args['mix_id'])
        else:
            mix = Mix.objects.get(slug=args['slug'])
    except Mix.DoesNotExist:
        raise Http404

    image = mix.get_image_url('400x400')
    audio_url = mix.get_stream_path()
    mix_url = mix.get_absolute_url()
    default = _getPayload(request)
    extras = {
        "description": mix.description.replace('
', '\n'),
        "title": mix.title,
        "image_url": image,
        "audio_url": audio_url,
        "mix_url": 'http://%s%s' % (Site.objects.get_current().domain, mix_url)
    }
    payload = dict(default.items() + extras.items())
    response = render_to_response(
        'inc/facebook/mix.html',
        payload,
        context_instance=RequestContext(request)
    )
    return response

def playlist(request, args):
    try:
        playlist = Playlist.objects.get(slug=args['slug'])
    except Playlist.DoesNotExist:
        raise Http404

    site_url = settings.DEBUG_URL if settings.DEBUG else Site.objects.get_current().domain
    image = playlist.get_image_url('400x400')
    playlist_url = "http://%s%s" % (site_url, playlist.get_absolute_url())
    default = _getPayload(request)
    extras = {
        "description": "Deep South Sounds Playlist by %s" % playlist.user.get_nice_name(),
        "title": playlist.name,
        "image_url": image,
        "playlist_url": playlist_url
    }
    payload = dict(default.items() + extras.items())
    response = render_to_response(
        'inc/facebook/playlist.html',
        payload,
        context_instance=RequestContext(request)
    )
    print response.content
    return response

def user(request, args):
    try:
        user = UserProfile.objects.get(slug=args['user_id'])
    except UserProfile.DoesNotExist:
        raise Http404

    image = user.get_avatar_image()
    profile_url = user.get_profile_url()
    default = _getPayload(request)
    extras = {
        "title": user.get_nice_name(),
        "description": user.get_profile_description().replace('
', '\n'),
        "profile_url": wrap_full(profile_url),
        "image_url": image,
    }
    payload = dict(default.items() + extras.items())
    response = render_to_response(
        'inc/facebook/user.html',
        payload,
        context_instance=RequestContext(request)
    )
    return response

def index(request):
    response = render_to_response(
        "inc/facebook/index.html",
        _getPayload(request),
        context_instance=RequestContext(request))
    return response

def social_redirect(request):
    try:
        resolver = resolve('/social' + request.path)
        if resolver is not None:
            logger.debug("Resolver successfully resolved")
            return resolver.func(request, resolver.kwargs)
        else:
            logger.debug("No resolver found for: $%s" % request.path)
    except Http404:
        logger.debug("404 on resolver: $%s" % request.path)
        return index(request)
    except Exception, ex:
        logger.debug("Unhandled exception in social_redirect: $%s" % ex)
        return index(request)

def post_like(request, mix):
    try:
        tokens = SocialToken.objects.filter(
            account__user=request.user,
            account__provider='facebook')
        for token in tokens:
            url = 'https://graph.facebook.com/%s/og.likes' % token.account.uid
            values = {
                'access_token': token.token,
                'object': mix.get_full_url(),
            }
            response = requests.post(url, data=values)
            if response.status_code == 200:
                print "Returned %s" % response.json
                return response.json['id']
            else:
                print "Returned status code of %s" % response.status_code
    except urllib2.HTTPError, httpEx:
        print httpEx.message
    except Exception, ex:
        print ex.message
    return ""

def delete_like(request, uid):
    try:
        tokens = SocialToken.objects.filter(account__user=request.user, account__provider='facebook')
        for token in tokens:
            url = "https://graph.facebook.com/%s" % uid
            values = {
                'access_token': token.token,
            }
            response = requests.delete(url, data=values)
            return response
    except Exception, ex:
        print "Error talking with facebook: %s" % ex.message

<|endoftext|>


<commit_before>from buildbot.steps import shell

__all__ = (
    "DatabaseBackupRestore", "CriticalTest", "DatabaseBackupRestore",
    "AptitudeInstall", "DatabaseMigrate", "GitSetVersion", "BuildDebianPackage",
)

class CriticalShellCommand(shell.ShellCommand):
    warnOnFailure = 1
    flunkOnFailure = 1
    haltOnFailure = 1

class CriticalTest(shell.Test):
    name = "test"
    description = ["running tests"]
    descriptionDone = ["tested"]
    warnOnFailure = 1
    flunkOnFailure = 1
    haltOnFailure = 1

class DatabaseBackupRestore(CriticalShellCommand):
    name = "restore backup"
    description = ["restoring backup"]
    descriptionDone = ["backup restored"]
    command = ["citools", "restore_backup"]

    def __init__(self, citools_config, command=None, **kwargs):
        if not command:
            command = self.command
            command.append("-c")
            command.append(citools_config)

        super(DatabaseBackupRestore, self).__init__(command=command, **kwargs)

class AptitudeInstall(CriticalShellCommand):
    name = "install package"
    description = ["installing package"]
    descriptionDone = ["package installed"]
    command = ["aptitude", "install"]

    def __init__(self, package_name, command=None, **kwargs):
        if not command:
            command = self.command
            command.append(package_name)
        super(AptitudeInstall, self).__init__(command=command, **kwargs)

class DatabaseMigrate(CriticalShellCommand):
    name = "migrating database"
    description = ["migrating database"]
    descriptionDone = ["database migrated"]
    command = None

    def __init__(self, manage_command, command=None, **kwargs):
        if not command:
            command = [manage_command, "migrate"]
        super(DatabaseMigrate, self).__init__(command=command, **kwargs)

class GitSetVersion(CriticalShellCommand):
    name = "update version"
    description = ["setting version"]
    descriptionDone = ["version set"]
    command = ["python", "setup.py", "compute_version_git"]

class BuildDebianPackage(CriticalShellCommand):
    name = "build debian package"
    description = ["building debian package"]
    descriptionDone = ["package build"]
    command = ["python", "setup.py", "bdist_deb"]
<commit_msg>Fixed super() calls<commit_after>from buildbot.steps import shell

__all__ = (
    "DatabaseBackupRestore", "CriticalTest", "DatabaseBackupRestore",
    "AptitudeInstall", "DatabaseMigrate", "GitSetVersion", "BuildDebianPackage",
)

class CriticalShellCommand(shell.ShellCommand):
    warnOnFailure = 1
    flunkOnFailure = 1
    haltOnFailure = 1

class CriticalTest(shell.Test):
    name = "test"
    description = ["running tests"]
    descriptionDone = ["tested"]
    warnOnFailure = 1
    flunkOnFailure = 1
    haltOnFailure = 1

class DatabaseBackupRestore(CriticalShellCommand):
    name = "restore backup"
    description = ["restoring backup"]
    descriptionDone = ["backup restored"]
    command = ["citools", "restore_backup"]

    def __init__(self, citools_config, command=None, **kwargs):
        if not command:
            command = [i for i in self.command]
            command.append("-c")
            command.append(citools_config)

        CriticalShellCommand.__init__(self, command=command, **kwargs)

class AptitudeInstall(CriticalShellCommand):
    name = "install package"
    description = ["installing package"]
    descriptionDone = ["package installed"]
    command = ["aptitude", "install"]

    def __init__(self, package_name, command=None, **kwargs):
        if not command:
            command = [i for i in self.command]
            command.append(package_name)
        CriticalShellCommand.__init__(self, command=command, **kwargs)

class DatabaseMigrate(CriticalShellCommand):
    name = "migrating database"
    description = ["migrating database"]
    descriptionDone = ["database migrated"]
    command = None

    def __init__(self, manage_command, command=None, **kwargs):
        if not command:
            command = [manage_command, "migrate"]
        CriticalShellCommand.__init__(self, command=command, **kwargs)

class GitSetVersion(CriticalShellCommand):
    name = "update version"
    description = ["setting version"]
    descriptionDone = ["version set"]
    command = ["python", "setup.py", "compute_version_git"]

class BuildDebianPackage(CriticalShellCommand):
    name = "build debian package"
    description = ["building debian package"]
    descriptionDone = ["package build"]
    command = ["python", "setup.py", "bdist_deb"]
<|endoftext|>


<commit_before>import sys
import pandas as pd
from ..model.Genre import Genre
from ..model.Movie import Movie
from ..model.Rating import Rating
from ..model.User import User
from sqlalchemy import *
from sqlalchemy.orm import sessionmaker
from recommender.model.Base import Base
from recommender.util.Search import binary_search

class Database:

    session = None

    @staticmethod
    def initialize():
        if Database.session is None:
            database_engine = create_engine('mysql+mysqldb://foo:Passw0rd!@172.18.0.2/recommendation')
            database_engine.echo = True

            ## Create model tables
            Base.metadata.create_all(database_engine)

            ## Create session
            session = sessionmaker()
            session.configure(bind=database_engine)
            Database.session = session()

    @staticmethod
    def load_data(file_name):
        data = pd.read_csv(file_name, delimiter=',')
        return data

    @staticmethod
    def setup():
        # http://docs.sqlalchemy.org/en/latest/core/engines.html
        root_path = "/opt/ml-latest-small"
        try:
            # Insert Movies
            print("Inserting Movies")
            file_name = root_path + "/movies.csv"
            data = Database.load_data(file_name)

            num_rows = len(data.values)
            for index, row in data.iterrows():
                list_genres = []
                # Check whether genre's have already been commited to the database
                genres = row['genres'].split('|')
                for genre in genres:
                    genre_object = Database.session.query(Genre).filter(Genre.name == genre).first()
                    if genre_object is None:
                        genre_object = Genre(name=genre)
                        Database.session.add(genre_object)
                        Database.session.commit()
                    list_genres.append(genre_object)

                movie = Movie(movieId=row['movieId'], title=row['title'])
                movie.genres = list_genres
                Database.session.add(movie)
                progress = (float(index) / float(num_rows))*100
                sys.stdout.write("Progress: %f%%    \r" % progress)
                sys.stdout.flush()

            Database.session.commit()
            movies = Database.session.query(Movie).all()

            print("Preparing movies for ratings")
            movie_ids = [movie.movieId for movie in movies]
            result = all(movie_ids[i] <= movie_ids[i + 1] for i in xrange(len(movie_ids) - 1))
            if not result:
                print("Movie ids are not sorted")
                exit(1)

            print("Finished inserting movies")
            users = []
            # ## Insert Ratings
            file_name = root_path + "/ratings.csv"
            data = Database.load_data(file_name)
            num_rows = len(data.values)

            print("Preparing Users")
            last_user_id = - 1
            ## Prep ratings insert
            for index, row in data.iterrows():
                ## New User
                if row['userId'] != last_user_id:
                    users.append(User(userId=row['userId']))

                last_user_id = row['userId']
                progress = (float(index) / float(num_rows)) * 100
                sys.stdout.write("Progress: %f%%    \r" % progress)
                sys.stdout.flush()

            Database.session.bulk_save_objects(users)
            Database.session.commit()
            users = Database.session.query(User).all()

            print("Configuring Ratings")
            ratings = []
            for index, row in data.iterrows():
                ## Find user and movie
                user = users[int(row['userId']) - 1]
                if user.userId != row['userId']:
                    print("User index didn't match up")
                    exit(1)

                movie_id = binary_search(movie_ids, int(row['movieId']))
                movie = movies[movie_id]
                if movie.movieId != row['movieId']:
                    print("Movie index didn't match up")
                    exit(1)

                ## Create new rating
                rating = Rating(value=row['rating'], timestamp=row['timestamp'], user_id=user.id, movie_id=movie.movieId)
                ratings.append(rating)
                #movie.ratings.append(rating)
                #user.ratings.append(rating)
                progress = (float(index) / float(num_rows)) * 100
                sys.stdout.write("Progress: %f%%    \r" % progress)
                sys.stdout.flush()

            ## Because the move and user objects are queried objects the commit will automatically
            ## add the rating to the ratings table at this point
            Database.session.bulk_save_objects(ratings)
            Database.session.commit()
            print("Finished Database setup")

        except Exception as e:
            print("Unable to insert data", e)
<commit_msg>fixed database rating setup<commit_after>import sys
import pandas as pd
from ..model.Genre import Genre
from ..model.Movie import Movie
from ..model.Rating import Rating
from ..model.User import User
from sqlalchemy import *
from sqlalchemy.orm import sessionmaker
from recommender.model.Base import Base
from recommender.util.Search import binary_search

class Database:

    session = None

    @staticmethod
    def initialize():
        if Database.session is None:
            database_engine = create_engine('mysql+mysqldb://foo:Passw0rd!@172.18.0.2/recommendation')
            database_engine.echo = True

            ## Create model tables
            Base.metadata.create_all(database_engine)

            ## Create session
            session = sessionmaker()
            session.configure(bind=database_engine)
            Database.session = session()

    @staticmethod
    def load_data(file_name):
        data = pd.read_csv(file_name, delimiter=',')
        return data

    @staticmethod
    def setup():
        # http://docs.sqlalchemy.org/en/latest/core/engines.html
        root_path = "/opt/ml-latest-small"
        try:
            # Insert Movies
            print("Inserting Movies")
            file_name = root_path + "/movies.csv"
            data = Database.load_data(file_name)

            num_rows = len(data.values)
            for index, row in data.iterrows():
                list_genres = []
                # Check whether genre's have already been commited to the database
                genres = row['genres'].split('|')
                for genre in genres:
                    genre_object = Database.session.query(Genre).filter(Genre.name == genre).first()
                    if genre_object is None:
                        genre_object = Genre(name=genre)
                        Database.session.add(genre_object)
                        Database.session.commit()
                    list_genres.append(genre_object)

                movie = Movie(movieId=row['movieId'], title=row['title'])
                movie.genres = list_genres
                Database.session.add(movie)
                progress = (float(index) / float(num_rows))*100
                sys.stdout.write("Progress: %f%%    \r" % progress)
                sys.stdout.flush()

            Database.session.commit()
            movies = Database.session.query(Movie).all()

            print("Preparing movies for ratings")
            movie_ids = [movie.movieId for movie in movies]
            result = all(movie_ids[i] <= movie_ids[i + 1] for i in xrange(len(movie_ids) - 1))
            if not result:
                print("Movie ids are not sorted")
                exit(1)

            print("Finished inserting movies")
            users = []
            # ## Insert Ratings
            file_name = root_path + "/ratings.csv"
            data = Database.load_data(file_name)
            num_rows = len(data.values)

            print("Preparing Users")
            last_user_id = - 1
            ## Prep ratings insert
            for index, row in data.iterrows():
                ## New User
                if row['userId'] != last_user_id:
                    users.append(User(userId=row['userId']))

                last_user_id = row['userId']
                progress = (float(index) / float(num_rows)) * 100
                sys.stdout.write("Progress: %f%%    \r" % progress)
                sys.stdout.flush()

            Database.session.bulk_save_objects(users)
            Database.session.commit()
            users = Database.session.query(User).all()

            print("Configuring Ratings")
            ratings = []
            for index, row in data.iterrows():
                ## Find user and movie
                user = users[int(row['userId']) - 1]
                if user.userId != row['userId']:
                    print("User index didn't match up")
                    exit(1)

                movie_id = binary_search(movie_ids, int(row['movieId']))
                movie = movies[movie_id]
                if movie.movieId != row['movieId']:
                    print("Movie index didn't match up")
                    exit(1)

                ## Create new rating
                rating = Rating(value=row['rating'], timestamp=row['timestamp'], user_id=user.id, movie_id=movie.id)
                ratings.append(rating)
                progress = (float(index) / float(num_rows)) * 100
                sys.stdout.write("Progress: %f%%    \r" % progress)
                sys.stdout.flush()

            ## Because the move and user objects are queried objects the commit will automatically
            ## add the rating to the ratings table at this point
            Database.session.bulk_save_objects(ratings)
            Database.session.commit()
            print("Finished Database setup")

        except Exception as e:
            print("Unable to insert data", e)
<|endoftext|>


<commit_before>import chainer.functions as F
from chainer import Variable, FunctionSet, optimizers
from chainer.links import caffe
from chainer import computational_graph
from chainer import cuda
from chainer import optimizers
from chainer import serializers
from .tensor import *
from .network import *
from .deel import *
import json
import os
import multiprocessing
import threading
import time
import six
import numpy as np
import os.path
from PIL import Image
#import six.moves.cPickle as pickle
from six.moves import queue
import pickle
import hashlib
import datetime
import sys
import random
from functools import cmp_to_key

"""Input something to context tensor"""
def Input(x):

	if isinstance(x,str):
		root, ext = os.path.splitext(x)
		if ext=='.png' or ext=='.jpg' or ext=='.jpeg' or ext=='.gif':
			img = Image.open(x)
			t = ImageTensor(img)
			t.use()
		elif ext=='.txt':
			print("this is txt")

	return t

def concat(x,y,train='on'):
	xdim = 1
	xp=Deel.xp
	if Deel.gpu>=0:
		x = cuda.to_cpu(x)
		y= cuda.to_cpu(y)
	x = x.copy()
	y = y.copy()
	for n in x.shape:
		xdim *= n
	if len(x.shape)>1:
		x = x.reshape((xdim,))
	ydim=1
	for n in y.shape:
		ydim *= n
	if len(y.shape)>1:
		y = y.reshape((ydim,))
	z = np.r_[x,y]

	z = xp.asarray(z,dtype=xp.float32)

	return z

def load(path, root):
	tuples = []
	for line in open(path):
		pair = line.strip().split()
		tuples.append((os.path.join(root, pair[0]), np.int32(pair[1])))
	return tuples

def read_image(path, center=False, flip=False):
	cropwidth = 256 - ImageNet.in_size
	try:
		image = np.asarray(Image.open(path)).transpose(2, 0, 1)
		if center:
			top = left = cropwidth / 2
		else:
			top = random.randint(0, cropwidth - 1)
			left = random.randint(0, cropwidth - 1)
		bottom = ImageNet.in_size + top
		right = ImageNet.in_size + left
		image = image[:, top:bottom, left:right].astype(np.float32)
		image -= ImageNet.mean_image[:, top:bottom, left:right]
		image /= 255
		if flip and random.randint(0, 1) == 0:
			return image[:, :, ::-1]
		else:
			return image
	except ValueError:
		print("---- Read_Image Value Error ----")
		print(path)

		return None

def batch_read_and_feed(batch):
	batchsize = BatchTrainer.batchsize
	in_size = BatchTrainer.in_size
	x_batch = np.ndarray(
		(batchsize, 3, in_size, in_size), dtype=np.float32)
	y_batch = np.ndarray((batchsize,), dtype=np.int32)
	i=0
	for data in batch:
		path,label = data
		#img = read_image (path, False, True)
		img = Image.open(path).convert('RGB')
		img = filter(np.asarray(img),flip=True,center=False)
		if img is not None:
			x_batch[i] = img
			y_batch[i] = label
			i+=1

	return x_batch,y_batch

def feed_data():
	global optimizer_lr,checkout
	# Data feeder
	i = 0
	count = 0
	in_size = BatchTrainer.in_size
	batchsize = BatchTrainer.batchsize
	val_batchsize = BatchTrainer.val_batchsize
	train_list = BatchTrainer.train_list
	val_list = BatchTrainer.val_list

	x_batch = np.ndarray(
		(batchsize, 3, in_size, in_size), dtype=np.float32)
	y_batch = np.ndarray((batchsize,), dtype=np.int32)

	val_x_batch = np.ndarray(
		(val_batchsize, 3, in_size, in_size), dtype=np.float32)
	val_y_batch = np.ndarray((val_batchsize,), dtype=np.int32)

	batch_pool = [None] * batchsize
	val_batch_pool = [None] * val_batchsize
	pool = multiprocessing.Pool(BatchTrainer.loaderjob)
	BatchTrainer.data_q.put('train')

	thleads = 1
	for epoch in six.moves.range(1, 1 + Deel.epoch):
		print('epoch', epoch)
		print('learning rate=%f'%Deel.optimizer_lr)

		#perm = np.random.permutation(len(train_list))
		random.shuffle(train_list)

		for idx in range(0,len(train_list)-batchsize*thleads,batchsize*thleads):
			#paths, labels = train_list[idx:idx+batchsize]
			#print paths

			r=[pool.apply_async(batch_read_and_feed,
					(train_list[idx+i*batchsize:
								idx+i*batchsize+batchsize],))
					for i in range(thleads)]
			for res in r:
				x_batch,y_batch = res.get()
				BatchTrainer.data_q.put((x_batch, y_batch))

			count += 1
			if count % 3000 == 0:
				print("checkout")
				if checkout:
					checkout()
				BatchTrainer.data_q.put('val')
				j = 0
				#val_list=train_list[idx:idx+val_batchsize]
				#random.shuffle(val_list)
				"""for path, label in val_list:
					val_batch_pool[j] = pool.apply_async(
						read_image, (path, True, False))
					val_y_batch[j] = label
					j += 1

					if j == val_batchsize:
						for k, x in enumerate(val_batch_pool):
							val_x_batch[k] = x.get()
						BatchTrainer.data_q.put((val_x_batch.copy(), val_y_batch.copy()))
						j = 0
				"""
				r=[pool.apply_async(batch_read_and_feed,
					(val_list[i*val_batchsize:i*val_batchsize+val_batchsize],))
					for i in range(len(val_list)/val_batchsize)]
				for res in r:
					val_x_batch,val_y_batch = res.get()
					BatchTrainer.data_q.put((val_x_batch.copy(), val_y_batch.copy()))

				BatchTrainer.data_q.put('train')
		Deel.optimizer_lr *= 0.98

	pool.close()
	pool.join()
	BatchTrainer.data_q.put('end')

def log_result():
	# Logger
	train_count = 0
	train_cur_loss = 0
	train_cur_accuracy = 0
	begin_at = time.time()
	val_batchsize = BatchTrainer.val_batchsize
	val_begin_at = None
	while True:
		result = BatchTrainer.res_q.get()
		if result == 'end':
			break
		elif result == 'train':
			train = True
			if val_begin_at is not None:
				begin_at += time.time() - val_begin_at
				val_begin_at = None
			continue
		elif result == 'val':
			train = False
			val_count = val_loss = val_accuracy = 0
			val_begin_at = time.time()
			continue

		loss, accuracy = result
		if train:
			train_count += 1
			duration = time.time() - begin_at
			throughput = train_count * BatchTrainer.batchsize / duration
			if train_count % 100 == 0:
				print(
					'\rtrain {} updates ({} samples) time: {} ({} images/sec)'
					.format(train_count, train_count * BatchTrainer.batchsize,
						datetime.timedelta(seconds=duration), throughput))

			train_cur_loss += loss
			train_cur_accuracy += accuracy
			if train_count % 1000 == 0:
				mean_loss = train_cur_loss / 1000
				mean_error = 1 - train_cur_accuracy / 1000
				print(json.dumps({'type': 'train', 'iteration': train_count,
								  'error': mean_error, 'loss': mean_loss}))
				sys.stdout.flush()
				train_cur_loss = 0
				train_cur_accuracy = 0
		else:
			val_count += val_batchsize
			print("valc",val_count)
			duration = time.time() - val_begin_at
			throughput = val_count / duration
			print(
				'\rval   {} batches ({} samples) time: {} ({} images/sec)'
				.format(val_count / val_batchsize, val_count,
						datetime.timedelta(seconds=duration), throughput))

			val_loss += loss
			val_accuracy += accuracy
			if val_count % 1024 ==0:
				mean_loss = val_loss *val_batchsize / 1024
				mean_error = 1 - val_accuracy  * val_batchsize/ 1024
				val_loss=0
				val_accuracy=0
				print(json.dumps({'type': 'val', 'iteration': train_count,
								  'error': mean_error, 'loss': mean_loss}))
def train_loop():
	global workout
	train=True
	Deel.trainCount=0
	while True:
		while BatchTrainer.data_q.empty():
			time.sleep(0.1)
		inp = BatchTrainer.data_q.get()
		if inp == 'end':  # quit
			BatchTrainer.res_q.put('end')
			break
		elif inp == 'train':  # restart training
			BatchTrainer.res_q.put('train')
			train=True
			continue
		elif inp == 'val':  # start validation
			BatchTrainer.res_q.put('val')
			train=False
			continue
		#train=False

		Deel.train = train

		volatile = 'off' if train else 'on'
		#volatile='on'

		if BatchTrainer.minibatch:
			_ax = Deel.xp.asarray(inp[0])
			_at = Deel.xp.asarray(inp[1])
		else:
			_ax = Deel.xp.asarray(inp[0][0])
			_at = Deel.xp.asarray(inp[1][0])

		if Deel.defferedTuning:
			_x =Variable(_ax, volatile='on')
			_t =Variable(_at, volatile='off')
		else:
			_x =Variable(_ax, volatile=volatile)
			_t =Variable(_at, volatile=volatile)

		x = ChainerTensor(_x)
		t = ChainerTensor(_t)

		if train:
			loss,accuracy = workout(x,t)

		Deel.trainCount+=1

		BatchTrainer.res_q.put((float(loss), float(accuracy)))
		del _x,_t
		del x,t
		del _ax,_at

def cnn_lstm_trainer(workout):
	Deel.trainCount=0
	while True:

		train_list = BatchTrainer.lstm_train

		for i in range(len(train_list)):
			x = Deel.xp.asarray([read_image(train_list[i][0],False,True)])
			x = ChainerTensor(Variable(x,volatile='on'))
			x.use()

			result = workout(x,train_list[i][1])

			Deel.trainCount+=1

def isImageFile(path):
	root, ext = os.path.splitext(path)
	if ext=='.gif':
		return True
	if ext=='.png':
		return True
	if ext=='.jpeg':
		return True
	if ext=='.jpg':
		return True
	return False

def InputBatch(train='data/train.txt',val='data/test.txt',minibatch=True):
	Deel.train=train
	Deel.val = val
	BatchTrainer.minibatch=minibatch

	root, ext = os.path.splitext(train)
	if ext==".txt":
		BatchTrainer.train_list = load(Deel.train,Deel.root)
		BatchTrainer.val_list = load(Deel.val,Deel.root)
		BatchTrainer.in_size=ImageNet.in_size
		BatchTrainer.mode='CNN'
	if ext==".tsv":
		lstm_pairs=[]
		for line in  open(train).readlines():
			data = line.rstrip().split('\t')
			lstm_pairs.append( data)
		BatchTrainer.lstm_train = lstm_pairs
		if isImageFile(lstm_pairs[0][0]):
			BatchTrainer.in_size=ImageNet.in_size
			BatchTrainer.mode='CNN-LSTM'
		else:
			BatchTrainer.mode='LSTM'

		vocab={}
		for i in range(len(lstm_pairs)):
			word = lstm_pairs[i][1]
			for char in word:
				if char not in vocab:
					vocab[char] = len(vocab)
		BatchTrainer.vocab = vocab

def BatchTrain(callback,callback2=None):
	global workout,checkout
	trainer = BatchTrainer()

	if BatchTrainer.mode=='CNN':
		feeder = threading.Thread(target=feed_data)
		feeder.daemon = True
		feeder.start()
		logger = threading.Thread(target=log_result)
		logger.daemon = True
		logger.start()

		workout = callback
		checkout=callback2

		train_loop()
		feeder.join()
		logger.join()
	elif BatchTrainer.mode=='CNN-LSTM':
		cnn_lstm_trainer(callback)

def Show(x=None):
	if x==None:
		x = Tensor.context

	x.show()

def ShowLabels(x=None):
	if x==None:
		x = Tensor.context

	t = LabelTensor(x)

	t.show()
	return t

def GetLabels(x=None):
	if x==None:
		x = Tensor.context

	t = LabelTensor(x)

	return t.content

def Output(x=None,num_of_candidate=5):
	if x==None:
		x = Tensor.context

	out = x.output

	out.sort(key=cmp_to_key(lambda a, b: ((a[0] > b[0]) - (a[0] < b[0]))), reverse=True)
	for rank, (score, name) in enumerate(out[:num_of_candidate], start=1):
		print('#%d | %s | %4.1f%%' % (rank, name, score * 100))

<commit_msg>hosfix<commit_after>import chainer.functions as F
from chainer import Variable, FunctionSet, optimizers
from chainer.links import caffe
from chainer import computational_graph
from chainer import cuda
from chainer import optimizers
from chainer import serializers
from .tensor import *
from .network import *
from .deel import *
import json
import os
import multiprocessing
import threading
import time
import six
import numpy as np
import os.path
from PIL import Image
#import six.moves.cPickle as pickle
from six.moves import queue
import pickle
import hashlib
import datetime
import sys
import random
from functools import cmp_to_key

"""Input something to context tensor"""
def Input(x):

	if isinstance(x,str):
		root, ext = os.path.splitext(x)
		if ext=='.png' or ext=='.jpg' or ext=='.jpeg' or ext=='.gif':
			img = Image.open(x)
			t = ImageTensor(img)
			t.use()
		elif ext=='.txt':
			print("this is txt")

	return t

def concat(x,y,train='on'):
	xdim = 1
	xp=Deel.xp
	if Deel.gpu>=0:
		x = cuda.to_cpu(x)
		y= cuda.to_cpu(y)
	x = x.copy()
	y = y.copy()
	for n in x.shape:
		xdim *= n
	if len(x.shape)>1:
		x = x.reshape((xdim,))
	ydim=1
	for n in y.shape:
		ydim *= n
	if len(y.shape)>1:
		y = y.reshape((ydim,))
	z = np.r_[x,y]

	z = xp.asarray(z,dtype=xp.float32)

	return z

def load(path, root):
	tuples = []
	for line in open(path):
		pair = line.strip().split()
		tuples.append((os.path.join(root, pair[0]), np.int32(pair[1])))
	return tuples

def read_image(path, center=False, flip=False):
	cropwidth = 256 - ImageNet.in_size
	try:
		image = np.asarray(Image.open(path)).transpose(2, 0, 1)
		if center:
			top = left = cropwidth / 2
		else:
			top = random.randint(0, cropwidth - 1)
			left = random.randint(0, cropwidth - 1)
		bottom = ImageNet.in_size + top
		right = ImageNet.in_size + left
		image = image[:, top:bottom, left:right].astype(np.float32)
		image -= ImageNet.mean_image[:, top:bottom, left:right]
		image /= 255
		if flip and random.randint(0, 1) == 0:
			return image[:, :, ::-1]
		else:
			return image
	except ValueError:
		print("---- Read_Image Value Error ----")
		print(path)

		return None

def batch_read_and_feed(batch):
	batchsize = BatchTrainer.batchsize
	in_size = BatchTrainer.in_size
	x_batch = np.ndarray(
		(batchsize, 3, in_size, in_size), dtype=np.float32)
	y_batch = np.ndarray((batchsize,), dtype=np.int32)
	i=0
	for data in batch:
		path,label = data
		#img = read_image (path, False, True)
		img = Image.open(path).convert('RGB')
		img = filter(np.asarray(img),flip=True,center=False)
		if img is not None:
			x_batch[i] = img
			y_batch[i] = label
			i+=1

	return x_batch,y_batch

def feed_data():
	global optimizer_lr,checkout
	# Data feeder
	i = 0
	count = 0
	in_size = BatchTrainer.in_size
	batchsize = BatchTrainer.batchsize
	val_batchsize = BatchTrainer.val_batchsize
	train_list = BatchTrainer.train_list
	val_list = BatchTrainer.val_list

	x_batch = np.ndarray(
		(batchsize, 3, in_size, in_size), dtype=np.float32)
	y_batch = np.ndarray((batchsize,), dtype=np.int32)

	val_x_batch = np.ndarray(
		(val_batchsize, 3, in_size, in_size), dtype=np.float32)
	val_y_batch = np.ndarray((val_batchsize,), dtype=np.int32)

	batch_pool = [None] * batchsize
	val_batch_pool = [None] * val_batchsize
	pool = multiprocessing.Pool(BatchTrainer.loaderjob)
	BatchTrainer.data_q.put('train')

	thleads = 1
	for epoch in six.moves.range(1, 1 + Deel.epoch):
		print('epoch', epoch)
		print('learning rate=%f'%Deel.optimizer_lr)

		#perm = np.random.permutation(len(train_list))
		random.shuffle(train_list)

		for idx in range(0,len(train_list)-batchsize*thleads,batchsize*thleads):
			#paths, labels = train_list[idx:idx+batchsize]
			#print paths

			r=[pool.apply_async(batch_read_and_feed,
					(train_list[idx+i*batchsize:
								idx+i*batchsize+batchsize],))
					for i in range(thleads)]
			for res in r:
				x_batch,y_batch = res.get()
				BatchTrainer.data_q.put((x_batch, y_batch))

			count += 1
			if count % 3000 == 0:
				print("checkout")
				if checkout:
					checkout()
				BatchTrainer.data_q.put('val')
				j = 0
				#val_list=train_list[idx:idx+val_batchsize]
				#random.shuffle(val_list)
				"""for path, label in val_list:
					val_batch_pool[j] = pool.apply_async(
						read_image, (path, True, False))
					val_y_batch[j] = label
					j += 1

					if j == val_batchsize:
						for k, x in enumerate(val_batch_pool):
							val_x_batch[k] = x.get()
						BatchTrainer.data_q.put((val_x_batch.copy(), val_y_batch.copy()))
						j = 0
				"""
				r=[pool.apply_async(batch_read_and_feed,
					(val_list[i*val_batchsize:i*val_batchsize+val_batchsize],))
					for i in range(len(val_list)/val_batchsize)]
				for res in r:
					val_x_batch,val_y_batch = res.get()
					BatchTrainer.data_q.put((val_x_batch.copy(), val_y_batch.copy()))

				BatchTrainer.data_q.put('train')
		Deel.optimizer_lr *= 0.98

	pool.close()
	pool.join()
	BatchTrainer.data_q.put('end')

def log_result():
	# Logger
	train_count = 0
	train_cur_loss = 0
	train_cur_accuracy = 0
	begin_at = time.time()
	val_batchsize = BatchTrainer.val_batchsize
	val_begin_at = None
	while True:
		result = BatchTrainer.res_q.get()
		if result == 'end':
			break
		elif result == 'train':
			train = True
			if val_begin_at is not None:
				begin_at += time.time() - val_begin_at
				val_begin_at = None
			continue
		elif result == 'val':
			train = False
			val_count = val_loss = val_accuracy = 0
			val_begin_at = time.time()
			continue

		loss, accuracy = result
		if train:
			train_count += 1
			duration = time.time() - begin_at
			throughput = train_count * BatchTrainer.batchsize / duration
			if train_count % 100 == 0:
				print(
					'\rtrain {} updates ({} samples) time: {} ({} images/sec)'
					.format(train_count, train_count * BatchTrainer.batchsize,
						datetime.timedelta(seconds=duration), throughput))

			train_cur_loss += loss
			train_cur_accuracy += accuracy
			if train_count % 1000 == 0:
				mean_loss = train_cur_loss / 1000
				mean_error = 1 - train_cur_accuracy / 1000
				print(json.dumps({'type': 'train', 'iteration': train_count,
								  'error': mean_error, 'loss': mean_loss}))
				sys.stdout.flush()
				train_cur_loss = 0
				train_cur_accuracy = 0
		else:
			val_count += val_batchsize
			print("valc",val_count)
			duration = time.time() - val_begin_at
			throughput = val_count / duration
			print(
				'\rval   {} batches ({} samples) time: {} ({} images/sec)'
				.format(val_count / val_batchsize, val_count,
						datetime.timedelta(seconds=duration), throughput))

			val_loss += loss
			val_accuracy += accuracy
			if val_count % 1024 ==0:
				mean_loss = val_loss *val_batchsize / 1024
				mean_error = 1 - val_accuracy  * val_batchsize/ 1024
				val_loss=0
				val_accuracy=0
				print(json.dumps({'type': 'val', 'iteration': train_count,
								  'error': mean_error, 'loss': mean_loss}))
def train_loop():
	global workout
	train=True
	Deel.trainCount=0
	while True:
		while BatchTrainer.data_q.empty():
			time.sleep(0.1)
		inp = BatchTrainer.data_q.get()
		if inp == 'end':  # quit
			BatchTrainer.res_q.put('end')
			break
		elif inp == 'train':  # restart training
			BatchTrainer.res_q.put('train')
			train=True
			continue
		elif inp == 'val':  # start validation
			BatchTrainer.res_q.put('val')
			train=False
			continue
		#train=False

		Deel.train = train

		volatile = 'off' if train else 'on'
		#volatile='on'

		if BatchTrainer.minibatch:
			_ax = Deel.xp.asarray(inp[0])
			_at = Deel.xp.asarray(inp[1])
		else:
			_ax = Deel.xp.asarray(inp[0][0])
			_at = Deel.xp.asarray(inp[1][0])

		if Deel.defferedTuning:
			_x =Variable(_ax, volatile='on')
			_t =Variable(_at, volatile='off')
		else:
			_x =Variable(_ax, volatile=volatile)
			_t =Variable(_at, volatile=volatile)

		x = ChainerTensor(_x)
		t = ChainerTensor(_t)

		loss,accuracy = workout(x,t)

		Deel.trainCount+=1

		BatchTrainer.res_q.put((float(loss), float(accuracy)))
		del _x,_t
		del x,t
		del _ax,_at

def cnn_lstm_trainer(workout):
	Deel.trainCount=0
	while True:

		train_list = BatchTrainer.lstm_train

		for i in range(len(train_list)):
			x = Deel.xp.asarray([read_image(train_list[i][0],False,True)])
			x = ChainerTensor(Variable(x,volatile='on'))
			x.use()

			result = workout(x,train_list[i][1])

			Deel.trainCount+=1

def isImageFile(path):
	root, ext = os.path.splitext(path)
	if ext=='.gif':
		return True
	if ext=='.png':
		return True
	if ext=='.jpeg':
		return True
	if ext=='.jpg':
		return True
	return False

def InputBatch(train='data/train.txt',val='data/test.txt',minibatch=True):
	Deel.train=train
	Deel.val = val
	BatchTrainer.minibatch=minibatch

	root, ext = os.path.splitext(train)
	if ext==".txt":
		BatchTrainer.train_list = load(Deel.train,Deel.root)
		BatchTrainer.val_list = load(Deel.val,Deel.root)
		BatchTrainer.in_size=ImageNet.in_size
		BatchTrainer.mode='CNN'
	if ext==".tsv":
		lstm_pairs=[]
		for line in  open(train).readlines():
			data = line.rstrip().split('\t')
			lstm_pairs.append( data)
		BatchTrainer.lstm_train = lstm_pairs
		if isImageFile(lstm_pairs[0][0]):
			BatchTrainer.in_size=ImageNet.in_size
			BatchTrainer.mode='CNN-LSTM'
		else:
			BatchTrainer.mode='LSTM'

		vocab={}
		for i in range(len(lstm_pairs)):
			word = lstm_pairs[i][1]
			for char in word:
				if char not in vocab:
					vocab[char] = len(vocab)
		BatchTrainer.vocab = vocab

def BatchTrain(callback,callback2=None):
	global workout,checkout
	trainer = BatchTrainer()

	if BatchTrainer.mode=='CNN':
		feeder = threading.Thread(target=feed_data)
		feeder.daemon = True
		feeder.start()
		logger = threading.Thread(target=log_result)
		logger.daemon = True
		logger.start()

		workout = callback
		checkout=callback2

		train_loop()
		feeder.join()
		logger.join()
	elif BatchTrainer.mode=='CNN-LSTM':
		cnn_lstm_trainer(callback)

def Show(x=None):
	if x==None:
		x = Tensor.context

	x.show()

def ShowLabels(x=None):
	if x==None:
		x = Tensor.context

	t = LabelTensor(x)

	t.show()
	return t

def GetLabels(x=None):
	if x==None:
		x = Tensor.context

	t = LabelTensor(x)

	return t.content

def Output(x=None,num_of_candidate=5):
	if x==None:
		x = Tensor.context

	out = x.output

	out.sort(key=cmp_to_key(lambda a, b: ((a[0] > b[0]) - (a[0] < b[0]))), reverse=True)
	for rank, (score, name) in enumerate(out[:num_of_candidate], start=1):
		print('#%d | %s | %4.1f%%' % (rank, name, score * 100))

<|endoftext|>


<commit_before><commit_msg>Erase validation count print<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging
# import json
import boto3
from core import ff_utils
from core.utils import Tibanna, ensure_list

LOG = logging.getLogger(__name__)
s3 = boto3.resource('s3')

def handler(event, context):
    '''
    this is generic function to run awsem workflow
    based on the data passed in

    workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name.
    Note multiple workflow_uuids can be available for an app_name
    (different versions of the same app could have a different uuid)
    '''
    # get incomming data
    input_file_list = event.get('input_files')
    app_name = event.get('app_name')
    print(app_name)
    workflow_uuid = event.get('workflow_uuid')
    output_bucket = event.get('output_bucket')
    tibanna_settings = event.get('_tibanna', {})
    # if they don't pass in env guess it from output_bucket
    env = tibanna_settings.get('env', '-'.join(output_bucket.split('-')[1:-1]))
    # tibanna provides access to keys based on env and stuff like that
    tibanna = Tibanna(env, s3_keys=event.get('s3_keys'), ff_keys=event.get('ff_keys'),
                      settings=tibanna_settings)

    args = dict()

    # get argument format & type info from workflow
    workflow_info = ff_utils.get_metadata(workflow_uuid, key=tibanna.ff_keys)
    LOG.info("workflow info  %s" % workflow_info)
    if 'error' in workflow_info.get('@type', []):
        raise Exception("FATAL, can't lookup workflow info for % fourfront" % workflow_uuid)

    # get cwl info from workflow_info
    for k in ['app_name', 'app_version', 'cwl_directory_url', 'cwl_main_filename', 'cwl_child_filenames']:
        logger.info(workflow_info.get(k))
        args[k] = workflow_info.get(k)

    # get format-extension map
    try:
        fp_schema = ff_utils.get_metadata("profiles/file_processed.json", key=tibanna.ff_keys)
        fe_map = fp_schema.get('file_format_file_extension')
    except Exception as e:
        LOG.error("Can't get format-extension map from file_processed schema. %s\n" % e)

    # processed file metadata
    output_files = []
    try:
        if 'arguments' in workflow_info:
            pf_meta = []
            for arg in workflow_info.get('arguments'):
                if (arg.get('argument_type') in
                   ['Output processed file', 'Output report file', 'Output QC file']):

                    of = dict()
                    of['workflow_argument_name'] = arg.get('workflow_argument_name')
                    of['type'] = arg.get('argument_type')
                    if 'argument_format' in arg:
                        # These are not processed files but report or QC files.
                        pf = ff_utils.ProcessedFileMetadata(file_format=arg.get('argument_format'))
                        try:
                            resp = pf.post(key=tibanna.ff_keys)  # actually post processed file metadata here
                            resp = resp.get('@graph')[0]
                            of['upload_key'] = resp.get('upload_key')
                            of['value'] = resp.get('uuid')
                        except Exception as e:
                            LOG.error("Failed to post Processed file metadata. %s\n" % e)
                            LOG.error("resp" + str(resp) + "\n")
                            raise e
                        of['format'] = arg.get('argument_format')
                        of['extension'] = fe_map.get(arg.get('argument_format'))
                        pf_meta.append(pf)
                    output_files.append(of)

    except Exception as e:
        LOG.error("output_files = " + str(output_files) + "\n")
        LOG.error("Can't prepare output_files information. %s\n" % e)
        raise e

    # create the ff_meta output info
    input_files = []
    for input_file in input_file_list:
        for idx, uuid in enumerate(ensure_list(input_file['uuid'])):
            input_files.append({'workflow_argument_name': input_file['workflow_argument_name'],
                                'value': uuid, 'ordinal': idx + 1})
    LOG.info("input_files is %s" % input_files)

    ff_meta = ff_utils.create_ffmeta_awsem(workflow_uuid, app_name, input_files,
                                           run_url=tibanna.settings.get('url', ''),
                                           output_files=output_files)

    LOG.info("ff_meta is %s" % ff_meta.__dict__)

    # store metadata so we know the run has started
    ff_meta.post_plain_wrf(key=tibanna.ff_keys)

    # input file args for awsem
    args['input_files'] = dict()
    for input_file in input_file_list:
        if isinstance(input_file['uuid'], str) and isinstance(input_file['object_key'], str):
            object_key = input_file['uuid'] + '/' + input_file['object_key']
        elif isinstance(input_file['uuid'], list) and isinstance(input_file['object_key'], list) and len(input_file['uuid']) == len(input_file['object_key']):
            object_key = [a + '/' + b for a, b in zip(input_file['uuid'], input_file['object_key'])]
        else:
            raise Exception("input_file uuid and object_key should match in their type and length (if lists)")
        args['input_files'].update({input_file['workflow_argument_name']: {
                                    'bucket_name': input_file['bucket_name'],
                                    'object_key': object_key}})

    LOG.info("input_file_args is %s" % args['input_files'])
    args['secondary_files'] = dict()   # temporary, later fill in based on the attachment information

    # parameters
    args['input_parameters'] = event.get('parameters')

    # output target
    args['output_target'] = dict()
    for of in ff_meta.output_files:
        arg_name = of.get('workflow_argument_name')
        if of.get('type') == 'Output processed file':
            args['output_target'][arg_name] = of.get('upload_key')
        else:
            args['output_target'][arg_name] = ff_meta.uuid + '/' + arg_name

    # output bucket
    args['output_S3_bucket'] = event.get('output_bucket')

    # let's not pass keys in plain text parameters
    return {"input_file_args": input_file_list,
            "ff_meta": ff_meta.as_dict(),
            'pf_meta': [meta.as_dict() for meta in pf_meta],
            "_tibanna": tibanna.as_dict(),
            "config": event.get("config"),
            "args": args
            }
<commit_msg>error loging on start_run_awsf<commit_after># -*- coding: utf-8 -*-
import logging
# import json
import boto3
from core import ff_utils
from core.utils import Tibanna, ensure_list

LOG = logging.getLogger(__name__)
s3 = boto3.resource('s3')

def handler(event, context):
    '''
    this is generic function to run awsem workflow
    based on the data passed in

    workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name.
    Note multiple workflow_uuids can be available for an app_name
    (different versions of the same app could have a different uuid)
    '''
    # get incomming data
    input_file_list = event.get('input_files')
    app_name = event.get('app_name')
    print(app_name)
    workflow_uuid = event.get('workflow_uuid')
    output_bucket = event.get('output_bucket')
    tibanna_settings = event.get('_tibanna', {})
    # if they don't pass in env guess it from output_bucket
    env = tibanna_settings.get('env', '-'.join(output_bucket.split('-')[1:-1]))
    # tibanna provides access to keys based on env and stuff like that
    tibanna = Tibanna(env, s3_keys=event.get('s3_keys'), ff_keys=event.get('ff_keys'),
                      settings=tibanna_settings)

    args = dict()

    # get argument format & type info from workflow
    workflow_info = ff_utils.get_metadata(workflow_uuid, key=tibanna.ff_keys)
    LOG.info("workflow info  %s" % workflow_info)
    if 'error' in workflow_info.get('@type', []):
        raise Exception("FATAL, can't lookup workflow info for % fourfront" % workflow_uuid)

    # get cwl info from workflow_info
    for k in ['app_name', 'app_version', 'cwl_directory_url', 'cwl_main_filename', 'cwl_child_filenames']:
        logger.info(workflow_info.get(k))
        args[k] = workflow_info.get(k)

    # get format-extension map
    try:
        fp_schema = ff_utils.get_metadata("profiles/file_processed.json", key=tibanna.ff_keys)
        fe_map = fp_schema.get('file_format_file_extension')
    except Exception as e:
        LOG.error("Can't get format-extension map from file_processed schema. %s\n" % e)

    # processed file metadata
    output_files = []
    try:
        if 'arguments' in workflow_info:
            pf_meta = []
            for arg in workflow_info.get('arguments'):
                if (arg.get('argument_type') in
                   ['Output processed file', 'Output report file', 'Output QC file']):

                    of = dict()
                    of['workflow_argument_name'] = arg.get('workflow_argument_name')
                    of['type'] = arg.get('argument_type')
                    if 'argument_format' in arg:
                        # These are not processed files but report or QC files.
                        pf = ff_utils.ProcessedFileMetadata(file_format=arg.get('argument_format'))
                        try:
                            resp = pf.post(key=tibanna.ff_keys)  # actually post processed file metadata here
                            resp = resp.get('@graph')[0]
                            of['upload_key'] = resp.get('upload_key')
                            of['value'] = resp.get('uuid')
                        except Exception as e:
                            LOG.error("Failed to post Processed file metadata. %s\n" % e)
                            LOG.error("resp" + str(resp) + "\n")
                            raise e
                        of['format'] = arg.get('argument_format')
                        of['extension'] = fe_map.get(arg.get('argument_format'))
                        pf_meta.append(pf)
                    output_files.append(of)

    except Exception as e:
        LOG.error("output_files = " + str(output_files) + "\n")
        LOG.error("Can't prepare output_files information. %s\n" % e)
        raise e

    # create the ff_meta output info
    input_files = []
    for input_file in input_file_list:
        for idx, uuid in enumerate(ensure_list(input_file['uuid'])):
            input_files.append({'workflow_argument_name': input_file['workflow_argument_name'],
                                'value': uuid, 'ordinal': idx + 1})
    LOG.info("input_files is %s" % input_files)

    ff_meta = ff_utils.create_ffmeta_awsem(workflow_uuid, app_name, input_files,
                                           run_url=tibanna.settings.get('url', ''),
                                           output_files=output_files)

    LOG.info("ff_meta is %s" % ff_meta.__dict__)

    # store metadata so we know the run has started
    ff_meta.post_plain_wrf(key=tibanna.ff_keys)

    # input file args for awsem
    args['input_files'] = dict()
    for input_file in input_file_list:
        if isinstance(input_file['uuid'], str) and isinstance(input_file['object_key'], str):
            object_key = input_file['uuid'] + '/' + input_file['object_key']
        elif isinstance(input_file['uuid'], list) and isinstance(input_file['object_key'], list) and len(input_file['uuid']) == len(input_file['object_key']):
            object_key = [a + '/' + b for a, b in zip(input_file['uuid'], input_file['object_key'])]
        else:
            raise Exception("input_file uuid and object_key should match in their type and length (if lists) : type{}{} length{}{}".format(type(input_file['uuid']), type(input_file['object_key']), len(input_file['uuid']), len(input_file['object_key'])))
        args['input_files'].update({input_file['workflow_argument_name']: {
                                    'bucket_name': input_file['bucket_name'],
                                    'object_key': object_key}})

    LOG.info("input_file_args is %s" % args['input_files'])
    args['secondary_files'] = dict()   # temporary, later fill in based on the attachment information

    # parameters
    args['input_parameters'] = event.get('parameters')

    # output target
    args['output_target'] = dict()
    for of in ff_meta.output_files:
        arg_name = of.get('workflow_argument_name')
        if of.get('type') == 'Output processed file':
            args['output_target'][arg_name] = of.get('upload_key')
        else:
            args['output_target'][arg_name] = ff_meta.uuid + '/' + arg_name

    # output bucket
    args['output_S3_bucket'] = event.get('output_bucket')

    # let's not pass keys in plain text parameters
    return {"input_file_args": input_file_list,
            "ff_meta": ff_meta.as_dict(),
            'pf_meta': [meta.as_dict() for meta in pf_meta],
            "_tibanna": tibanna.as_dict(),
            "config": event.get("config"),
            "args": args
            }
<|endoftext|>


<commit_before>from queue import Queue
from classes.specializedMatchers import DesperateMatcher
from classes.requesterThread import RequesterThread
from classes.plugin import Plugin
from collections import Counter
from collections import defaultdict
from html.parser import HTMLParser

class Desperate(object):
	# Desperate mode
	# search the cache for requests that match any of the fingerprints
	# in the supplied MD5 database 'fingerprints'
	def __init__(self):
		self.fingerprints = []
		self.matcher = DesperateMatcher
		self.matches = defaultdict(Counter)

	def add_fingerprints(self, fingerprints):
		self.fingerprints.extend(fingerprints)

	def set_cache(self, cache):
		self.cache = cache

	def crawl(self):
		ps = PageScraper(self.cache.host, self.cache)
		ps.run()

	def run(self):
		self.crawl()

		matcher = self.matcher(self.cache, self.fingerprints)
		for match in matcher.get_matches():
			version = match['output']
			cms = match['cms']
			self.matches[cms][version] += 1

	def get_matches(self):
		out = []
		for cms in self.matches:
			for version in self.matches[cms]:
				out.append( (self.matches[cms][version], cms, version) )

		out.sort()
		out.reverse()
		return [{'cms': i[1], 'version': i[2], 'count': i[0] } for i in out]

class LinkExtractor(HTMLParser):
	def __init__(self, strict):
		super().__init__(strict=strict)
		self.results = set()

	def get_results(self):
		return self.results

	def handle_starttag(self, tag, attrs):
		try:
			url = ''
			if tag == 'script' or tag == 'img':
				for attr in attrs:
					if attr[0] == 'src':  self.results.add(attr[1])
			if tag == 'link':
				for attr in attrs:
					if attr[0] == 'href': self.results.add(attr[1])
		except:
			pass

class PageScraper(Plugin):
	def __init__(self, host, cache):
		self.parser = LinkExtractor(strict=False)
		self.host = host
		self.cache = cache
		self.queue = Queue()
		self.results = Queue()
		self.threads = 4
		self.workers = []

	def run(self):
		for req in self.cache.get_responses():
			# only scrape pages that can contain links/references
			if 'text/html' in req.headers['content-type']:
				self.parser.feed(str(req.content))

				for i in self.parser.get_results():

					# ensure that only ressources located on the domain /sub-domain is requested
					if i.startswith('http'):
						parts = i.split('/')
						host = parts[2]

						# if the ressource is out side of the domain, skip it
						if not host in self.host.split('/')[2]:
							continue

						# else update the url so that it only contains the relative location
						else:
							i = '/'.join(parts[3:])

					self.queue.put( {"host": self.host, "url": i} )

		# add 'None' to queue - stops threads when no items are left
		for i in range(self.threads): self.queue.put( None )

		# start the threads
		for i in range(self.threads):
			w = RequesterThread(i, self.queue, self.cache, self.results)
			w.daemon = True
			self.workers.append(w)
			w.start()

		self.queue.join()

<commit_msg>Fixed crash if content-type is not present in response<commit_after>from queue import Queue
from classes.specializedMatchers import DesperateMatcher
from classes.requesterThread import RequesterThread
from classes.plugin import Plugin
from collections import Counter
from collections import defaultdict
from html.parser import HTMLParser

class Desperate(object):
	# Desperate mode
	# search the cache for requests that match any of the fingerprints
	# in the supplied MD5 database 'fingerprints'
	def __init__(self):
		self.fingerprints = []
		self.matcher = DesperateMatcher
		self.matches = defaultdict(Counter)

	def add_fingerprints(self, fingerprints):
		self.fingerprints.extend(fingerprints)

	def set_cache(self, cache):
		self.cache = cache

	def crawl(self):
		ps = PageScraper(self.cache.host, self.cache)
		ps.run()

	def run(self):
		self.crawl()

		matcher = self.matcher(self.cache, self.fingerprints)
		for match in matcher.get_matches():
			version = match['output']
			cms = match['cms']
			self.matches[cms][version] += 1

	def get_matches(self):
		out = []
		for cms in self.matches:
			for version in self.matches[cms]:
				out.append( (self.matches[cms][version], cms, version) )

		out.sort()
		out.reverse()
		return [{'cms': i[1], 'version': i[2], 'count': i[0] } for i in out]

class LinkExtractor(HTMLParser):
	def __init__(self, strict):
		super().__init__(strict=strict)
		self.results = set()

	def get_results(self):
		return self.results

	def handle_starttag(self, tag, attrs):
		try:
			url = ''
			if tag == 'script' or tag == 'img':
				for attr in attrs:
					if attr[0] == 'src':  self.results.add(attr[1])
			if tag == 'link':
				for attr in attrs:
					if attr[0] == 'href': self.results.add(attr[1])
		except:
			pass

class PageScraper(Plugin):
	def __init__(self, host, cache):
		self.parser = LinkExtractor(strict=False)
		self.host = host
		self.cache = cache
		self.queue = Queue()
		self.results = Queue()
		self.threads = 4
		self.workers = []

	def run(self):
		for req in self.cache.get_responses():
			# skip pages that do not set 'content-type'
			# these might be binaries
			if not 'content-tupe' in req.headers:
				continue

			# only scrape pages that can contain links/references
			if 'text/html' in req.headers['content-type']:
				self.parser.feed(str(req.content))

				for i in self.parser.get_results():

					# ensure that only ressources located on the domain /sub-domain is requested
					if i.startswith('http'):
						parts = i.split('/')
						host = parts[2]

						# if the ressource is out side of the domain, skip it
						if not host in self.host.split('/')[2]:
							continue

						# else update the url so that it only contains the relative location
						else:
							i = '/'.join(parts[3:])

					self.queue.put( {"host": self.host, "url": i} )

		# add 'None' to queue - stops threads when no items are left
		for i in range(self.threads): self.queue.put( None )

		# start the threads
		for i in range(self.threads):
			w = RequesterThread(i, self.queue, self.cache, self.results)
			w.daemon = True
			self.workers.append(w)
			w.start()

		self.queue.join()

<|endoftext|>