<commit_before>#!/usr/bin/python

# Define a NAT ASG and a Bastion ASG. Depends on a network stack to be defined.

import sys
import awacs.aws
import awacs.sts
import troposphere.ec2
import troposphere.iam
import troposphere.autoscaling
import troposphere as tp
from . import vpc # TODO TemplateBuilderBase should be in __init__

class ServicesTemplate(vpc.TemplateBuilderBase):

    BASTION_KEY_PARM_NAME = 'BastionKey'

    def __init__(self, name,
                 vpc_id,
                 vpc_cidr,
                 private_route_table_id,
                 public_subnet_ids,
                 description = '[REPLACEME]',
                 region = 'us-west-2',
                 nat_instance_type = 't2.micro',
                 bastion_instance_type = 't2.micro'):
        super(ServicesTemplate, self).__init__(name, description)

        self.vpc_id = vpc_id
        self.vpc_cidr = vpc_cidr
        self.subnet_ids = public_subnet_ids
        self.route_table_id = private_route_table_id
        self.region = region
        self.nat_instance_type = nat_instance_type
        self.bastion_instance_type = bastion_instance_type

        self.create_parameters()

        self.create_nat()
        self.create_bastion()

    def create_parameters(self):
        self.add_parameter(tp.Parameter(self.BASTION_KEY_PARM_NAME, Type = 'String'))

    def create_nat(self):
        nat_sg = self.create_nat_sg()
        nat_asg = self.create_nat_asg(nat_sg)
        # ENI?
        # IAM Profile?

    def create_bastion(self):
        bastion_sg = self.create_bastion_sg()
        bastion_asg = self.create_bastion_asg(bastion_sg)
        # EIP?
        # IAM Profile?

    def create_nat_asg(self, sg):
        profile = self.create_nat_iam_profile()
        lc = tp.autoscaling.LaunchConfiguration('NATLC',
                                                ImageId = tp.FindInMap(vpc.AMI_REGION_MAP_NAME, vpc.REF_REGION, 'NAT'),
                                                InstanceType = self.nat_instance_type,
                                                SecurityGroups = [tp.Ref(sg)],
                                                KeyName = tp.Ref(self.BASTION_KEY_PARM_NAME),
                                                IamInstanceProfile = tp.Ref(profile),
                                                AssociatePublicIpAddress = True,
                                                UserData = self._create_nat_userdata())
        asg = tp.autoscaling.AutoScalingGroup('NATASG',
                                              MinSize = 1, MaxSize = 1,
                                              LaunchConfigurationName = tp.Ref(lc),

                                              VPCZoneIdentifier = self.subnet_ids,
                                              Tags = tp.autoscaling.Tags(Name = 'NAT'))
        # TODO: Copy ALL default tags to AutoScalingTags
        self.add_resources([asg, lc])
        return asg

    def create_nat_iam_profile(self):
        role = tp.iam.Role('NATInstanceRole',
                           AssumeRolePolicyDocument = {
                               'Statement' : [ {
                                   'Effect' : 'Allow',
                                   'Principal' : { 'Service' : ['ec2.amazonaws.com'] },
                                   'Action' : ['sts:AssumeRole']
                               } ]
                           },
                           Policies = [
                               tp.iam.Policy(
                                   PolicyName = 'NATInstance',
                                   PolicyDocument = {
                                       'Statement' : [ {
                                           'Effect' : 'Allow',
                                           'Resource' : ['*'],
                                           'Action' : ['ec2:CreateRoute', 'ec2:DeleteRoute']
                                       } ]
                                   }
                               )
                           ])
        profile = tp.iam.InstanceProfile('NATInstanceProfile',
                                         Path = '/',
                                         Roles = [tp.Ref(role)])
        self.add_resources([role, profile])
        return profile

    def _create_nat_userdata(self):
        startup = [
            "#!/bin/bash",
            "yum update -y && yum install -y yum-cron && chkconfig yum-cron on",
            "INS_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`",
            "aws ec2 delete-route --destination-cidr-block 0.0.0.0/0 --route-table-id {} --region {}".format(self.route_table_id, self.region),
            "aws ec2 create-route --route-table-id {} --destination-cidr-block 0.0.0.0/0 --instance-id $INS_ID --region {}".format(self.route_table_id, self.region)
        ]
        return tp.Base64(tp.Join('\n', startup))

    def create_bastion_asg(self, sg):
        lc = tp.autoscaling.LaunchConfiguration('BastionLC',
                                                ImageId = tp.FindInMap(vpc.AMI_REGION_MAP_NAME, vpc.REF_REGION, 'BASTION'),
                                                InstanceType = self.bastion_instance_type,
                                                SecurityGroups = [tp.Ref(sg)],
                                                KeyName = tp.Ref(self.BASTION_KEY_PARM_NAME),
                                                AssociatePublicIpAddress = True,
                                                UserData = self._create_bastion_userdata())
        asg = tp.autoscaling.AutoScalingGroup('BastionASG',
                                              MinSize = 1, MaxSize = 1,
                                              LaunchConfigurationName = tp.Ref(lc),
                                              VPCZoneIdentifier = self.subnet_ids,
                                              Tags = tp.autoscaling.Tags(Name = 'Bastion'))
        # TODO: Copy ALL default tags to AutoScalingTags
        self.add_resources([asg, lc])
        return asg

    def _create_bastion_userdata(self):
        startup = [
            '#!/bin/bash',
            'yum update -y && yum install -y yum-cron && chkconfig yum-cron on'
        ]
        return tp.Base64(tp.Join('\n', startup))

    def create_nat_sg(self):
        sg = tp.ec2.SecurityGroup('NATSecurityGroup',
                                  GroupDescription = 'NAT Instance Security Group',
                                  SecurityGroupIngress = vpc.SecurityGroupRuleBuilder(self.vpc_cidr).any().rules(),
                                  VpcId = self.vpc_id,
                                  Tags = self.default_tags)
        self.add_resource(sg)
        return sg

    def create_bastion_sg(self):
        sg = tp.ec2.SecurityGroup('BastionSecurityGroup',
                                  GroupDescription = 'Bastion Instance Security Group',
                                  SecurityGroupIngress = vpc.SecurityGroupRuleBuilder(vpc.CIDR_ANY).ssh().rules(),
                                  VpcId = self.vpc_id,
                                  Tags = self.default_tags)
        self.add_resource(sg)
        return sg

if __name__ == '__main__':
    name = sys.argv[1] if len(sys.argv) > 1 else 'Test'
    vpc_id = sys.argv[2] if len(sys.argv) > 2 else 'vpc-deadbeef'
    vpc_cidr = sys.argv[3] if len(sys.argv) > 3 else '10.0.0.0/8'
    route_table = sys.argv[4] if len(sys.argv) > 4 else 'rtb-deadbeef'
    subnets = sys.argv[5:] if len(sys.argv) > 5 else ['subnet-deadbeef', 'subnet-cab4abba']
    print ServicesTemplate(name, vpc_id, vpc_cidr, route_table, subnets).to_json()
<commit_msg>turn detailed monitoring off<commit_after>#!/usr/bin/python

# Define a NAT ASG and a Bastion ASG. Depends on a network stack to be defined.

import sys
import awacs.aws
import awacs.sts
import troposphere.ec2
import troposphere.iam
import troposphere.autoscaling
import troposphere as tp
from . import vpc # TODO TemplateBuilderBase should be in __init__

class ServicesTemplate(vpc.TemplateBuilderBase):

    BASTION_KEY_PARM_NAME = 'BastionKey'

    def __init__(self, name,
                 vpc_id,
                 vpc_cidr,
                 private_route_table_id,
                 public_subnet_ids,
                 description = '[REPLACEME]',
                 region = 'us-west-2',
                 nat_instance_type = 't2.micro',
                 bastion_instance_type = 't2.micro'):
        super(ServicesTemplate, self).__init__(name, description)

        self.vpc_id = vpc_id
        self.vpc_cidr = vpc_cidr
        self.subnet_ids = public_subnet_ids
        self.route_table_id = private_route_table_id
        self.region = region
        self.nat_instance_type = nat_instance_type
        self.bastion_instance_type = bastion_instance_type

        self.create_parameters()

        self.create_nat()
        self.create_bastion()

    def create_parameters(self):
        self.add_parameter(tp.Parameter(self.BASTION_KEY_PARM_NAME, Type = 'String'))

    def create_nat(self):
        nat_sg = self.create_nat_sg()
        nat_asg = self.create_nat_asg(nat_sg)
        # ENI?
        # IAM Profile?

    def create_bastion(self):
        bastion_sg = self.create_bastion_sg()
        bastion_asg = self.create_bastion_asg(bastion_sg)
        # EIP?
        # IAM Profile?

    def create_nat_asg(self, sg):
        profile = self.create_nat_iam_profile()
        lc = tp.autoscaling.LaunchConfiguration('NATLC',
                                                ImageId = tp.FindInMap(vpc.AMI_REGION_MAP_NAME, vpc.REF_REGION, 'NAT'),
                                                InstanceType = self.nat_instance_type,
                                                SecurityGroups = [tp.Ref(sg)],
                                                KeyName = tp.Ref(self.BASTION_KEY_PARM_NAME),
                                                IamInstanceProfile = tp.Ref(profile),
                                                InstanceMonitoring = False,
                                                AssociatePublicIpAddress = True,
                                                UserData = self._create_nat_userdata())
        asg = tp.autoscaling.AutoScalingGroup('NATASG',
                                              MinSize = 1, MaxSize = 1,
                                              LaunchConfigurationName = tp.Ref(lc),

                                              VPCZoneIdentifier = self.subnet_ids,
                                              Tags = tp.autoscaling.Tags(Name = 'NAT'))
        # TODO: Copy ALL default tags to AutoScalingTags
        self.add_resources([asg, lc])
        return asg

    def create_nat_iam_profile(self):
        role = tp.iam.Role('NATInstanceRole',
                           AssumeRolePolicyDocument = {
                               'Statement' : [ {
                                   'Effect' : 'Allow',
                                   'Principal' : { 'Service' : ['ec2.amazonaws.com'] },
                                   'Action' : ['sts:AssumeRole']
                               } ]
                           },
                           Policies = [
                               tp.iam.Policy(
                                   PolicyName = 'NATInstance',
                                   PolicyDocument = {
                                       'Statement' : [ {
                                           'Effect' : 'Allow',
                                           'Resource' : ['*'],
                                           'Action' : ['ec2:CreateRoute', 'ec2:DeleteRoute']
                                       } ]
                                   }
                               )
                           ])
        profile = tp.iam.InstanceProfile('NATInstanceProfile',
                                         Path = '/',
                                         Roles = [tp.Ref(role)])
        self.add_resources([role, profile])
        return profile

    def _create_nat_userdata(self):
        startup = [
            "#!/bin/bash",
            "yum update -y && yum install -y yum-cron && chkconfig yum-cron on",
            "INS_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`",
            "aws ec2 delete-route --destination-cidr-block 0.0.0.0/0 --route-table-id {} --region {}".format(self.route_table_id, self.region),
            "aws ec2 create-route --route-table-id {} --destination-cidr-block 0.0.0.0/0 --instance-id $INS_ID --region {}".format(self.route_table_id, self.region)
        ]
        return tp.Base64(tp.Join('\n', startup))

    def create_bastion_asg(self, sg):
        lc = tp.autoscaling.LaunchConfiguration('BastionLC',
                                                ImageId = tp.FindInMap(vpc.AMI_REGION_MAP_NAME, vpc.REF_REGION, 'BASTION'),
                                                InstanceType = self.bastion_instance_type,
                                                SecurityGroups = [tp.Ref(sg)],
                                                KeyName = tp.Ref(self.BASTION_KEY_PARM_NAME),
                                                InstanceMonitoring = False,
                                                AssociatePublicIpAddress = True,
                                                UserData = self._create_bastion_userdata())
        asg = tp.autoscaling.AutoScalingGroup('BastionASG',
                                              MinSize = 1, MaxSize = 1,
                                              LaunchConfigurationName = tp.Ref(lc),
                                              VPCZoneIdentifier = self.subnet_ids,
                                              Tags = tp.autoscaling.Tags(Name = 'Bastion'))
        # TODO: Copy ALL default tags to AutoScalingTags
        self.add_resources([asg, lc])
        return asg

    def _create_bastion_userdata(self):
        startup = [
            '#!/bin/bash',
            'yum update -y && yum install -y yum-cron && chkconfig yum-cron on'
        ]
        return tp.Base64(tp.Join('\n', startup))

    def create_nat_sg(self):
        sg = tp.ec2.SecurityGroup('NATSecurityGroup',
                                  GroupDescription = 'NAT Instance Security Group',
                                  SecurityGroupIngress = vpc.SecurityGroupRuleBuilder(self.vpc_cidr).any().rules(),
                                  VpcId = self.vpc_id,
                                  Tags = self.default_tags)
        self.add_resource(sg)
        return sg

    def create_bastion_sg(self):
        sg = tp.ec2.SecurityGroup('BastionSecurityGroup',
                                  GroupDescription = 'Bastion Instance Security Group',
                                  SecurityGroupIngress = vpc.SecurityGroupRuleBuilder(vpc.CIDR_ANY).ssh().rules(),
                                  VpcId = self.vpc_id,
                                  Tags = self.default_tags)
        self.add_resource(sg)
        return sg

if __name__ == '__main__':
    name = sys.argv[1] if len(sys.argv) > 1 else 'Test'
    vpc_id = sys.argv[2] if len(sys.argv) > 2 else 'vpc-deadbeef'
    vpc_cidr = sys.argv[3] if len(sys.argv) > 3 else '10.0.0.0/8'
    route_table = sys.argv[4] if len(sys.argv) > 4 else 'rtb-deadbeef'
    subnets = sys.argv[5:] if len(sys.argv) > 5 else ['subnet-deadbeef', 'subnet-cab4abba']
    print ServicesTemplate(name, vpc_id, vpc_cidr, route_table, subnets).to_json()
<|endoftext|>


<commit_before># ex: set expandtab softtabstop=4 shiftwidth=4: -*- cpy-indent-level: 4; indent-tabs-mode: nil -*-
#
# Copyright (C) 2008,2009,2010  Contributor
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the EU DataGrid Software License.  You should
# have received a copy of the license with this program, and the
# license is published at
# http://eu-datagrid.web.cern.ch/eu-datagrid/license.html.
#
# THE FOLLOWING DISCLAIMER APPLIES TO ALL SOFTWARE CODE AND OTHER
# MATERIALS CONTRIBUTED IN CONNECTION WITH THIS PROGRAM.
#
# THIS SOFTWARE IS LICENSED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE AND ANY WARRANTY OF NON-INFRINGEMENT, ARE
# DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS
# SOFTWARE MAY BE REDISTRIBUTED TO OTHERS ONLY BY EFFECTIVELY USING
# THIS OR ANOTHER EQUIVALENT DISCLAIMER AS WELL AS ANY OTHER LICENSE
# TERMS THAT MAY APPLY.
""" Translates dotted quad strings into long integers """

from struct import pack, unpack
from socket import inet_aton, inet_ntoa
from ipaddr import IPv4Address

import sqlalchemy

class IPV4(sqlalchemy.types.TypeDecorator):
    """ A type to wrap IP addresses to and from the DB """

    impl = sqlalchemy.types.Integer
    impl.length = 9  # hardcoding for now, TODO: figure it out and fix

    def process_bind_param(self, value, engine):
        if value is None:
            ip = None
        elif isinstance(value, IPv4Address):
            ip = int(value)
        else:  # pragma: no cover
            raise TypeError("Unknown input type for IPv4 column: %s" % repr(value))
        return ip

    def process_result_value(self, value, engine):
        if value is None:
            return None
        else:
            # The explicit cast to int() is needed due to bug
            # http://code.google.com/p/ipaddr-py/issues/detail?id=70
            #
            # Background: when an IPv4Address object is initialized with a
            # string, the _ip field becomes an int(). However the database can
            # sometimes return the value as a long(), and although
            # IPv4Address(int(x)) == IPv4Address(long(x)), their hash() values
            # differ. Which means if one of the objects is used as a dictionary
            # key, using the other to do a lookup will fail.
            return IPv4Address(int(value))

    def copy(self):
        return IPV4(self.impl.length)
<commit_msg>IPV4 column: remove unused socket, inet imports<commit_after># ex: set expandtab softtabstop=4 shiftwidth=4: -*- cpy-indent-level: 4; indent-tabs-mode: nil -*-
#
# Copyright (C) 2008,2009,2010  Contributor
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the EU DataGrid Software License.  You should
# have received a copy of the license with this program, and the
# license is published at
# http://eu-datagrid.web.cern.ch/eu-datagrid/license.html.
#
# THE FOLLOWING DISCLAIMER APPLIES TO ALL SOFTWARE CODE AND OTHER
# MATERIALS CONTRIBUTED IN CONNECTION WITH THIS PROGRAM.
#
# THIS SOFTWARE IS LICENSED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE AND ANY WARRANTY OF NON-INFRINGEMENT, ARE
# DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS
# SOFTWARE MAY BE REDISTRIBUTED TO OTHERS ONLY BY EFFECTIVELY USING
# THIS OR ANOTHER EQUIVALENT DISCLAIMER AS WELL AS ANY OTHER LICENSE
# TERMS THAT MAY APPLY.
""" Translates dotted quad strings into long integers """
import sqlalchemy
from ipaddr import IPv4Address

class IPV4(sqlalchemy.types.TypeDecorator):
    """ A type to wrap IP addresses to and from the DB """

    impl = sqlalchemy.types.Integer
    impl.length = 9  # hardcoding for now, TODO: figure it out and fix

    def process_bind_param(self, value, engine):
        if value is None:
            ip = None
        elif isinstance(value, IPv4Address):
            ip = int(value)
        else:  # pragma: no cover
            raise TypeError("Unknown input type for IPv4 column: %s" % repr(value))
        return ip

    def process_result_value(self, value, engine):
        if value is None:
            return None
        else:
            # The explicit cast to int() is needed due to bug
            # http://code.google.com/p/ipaddr-py/issues/detail?id=70
            #
            # Background: when an IPv4Address object is initialized with a
            # string, the _ip field becomes an int(). However the database can
            # sometimes return the value as a long(), and although
            # IPv4Address(int(x)) == IPv4Address(long(x)), their hash() values
            # differ. Which means if one of the objects is used as a dictionary
            # key, using the other to do a lookup will fail.
            return IPv4Address(int(value))

    def copy(self):
        return IPV4(self.impl.length)
<|endoftext|>


<commit_before>#    vyconf.tree.referencetree._init__: package init file.
#
#    Copyright (C) 2014 VyOS Development Group <maintainers@vyos.net>
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#    USA

from vyconf.tree.referencetree.reference_node import (ReferenceNode,
    ReferenceNodeError)

from vyconf.tree.referencetree.reference_tree_loader import (ReferenceTreeLoader,
    ReferenceTreeLoaderError)
<commit_msg>Python3-friendly relative imports in referencetree.<commit_after>#    vyconf.tree.referencetree._init__: package init file.
#
#    Copyright (C) 2014 VyOS Development Group <maintainers@vyos.net>
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#    USA

from .reference_node import (ReferenceNode,
    ReferenceNodeError)

from .reference_tree_loader import (ReferenceTreeLoader,
    ReferenceTreeLoaderError)
<|endoftext|>


<commit_before>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from tensorflow.core.framework import attr_value_pb2
from tensorflow.core.framework import graph_pb2
from tensorflow.core.framework import node_def_pb2
from tensorflow.python.framework import graph_util
from tensorflow.python.framework import ops
from tensorflow.python.ops import variable_scope
from tensorflow.python.platform import gfile
from tensorflow.python.util import tf_contextlib
import tensorflow as tf
import os
import json
import copy

def _variable_creator(next_creator, **kwargs):
    kwargs["use_resource"] = False
    return next_creator(**kwargs)

@tf_contextlib.contextmanager
def variable_creator_scope():
    with variable_scope.variable_creator_scope(_variable_creator):
        yield

def process_grad(grad):
    if grad is not None:
        grad = ops.convert_to_tensor_or_indexed_slices(grad)
        if isinstance(grad, ops.IndexedSlices):
            # In IndexedSlices is not supported in java api, we have to convert it to
            # a dense tensor. This operation is potentially expensive, but there seems
            # no work around
            grad = tf.unsorted_segment_sum(grad.values, grad.indices,
                                           grad.dense_shape[0])
    return grad

def export_tf(sess, folder, inputs, outputs,
              generate_backward=False, allow_non_differentiable_input=True):
    """
    Export the frozen tensorflow graph as well as the inputs/outputs information
    to the folder for inference.

    This function will
    1. freeze the graph (replace all variables with constants)
    2. strip all unused node as specified by inputs and outputs
    3. add placeholder nodes as needed
    4. write the frozen graph and inputs/outputs names to the folder

    Note: There should not be any queuing operation between inputs and outputs

    :param sess: tensorflow session holding the variables to be saved
    :param folder: the folder where graph file and inputs/outputs information are saved
    :param inputs: a list of tensorflow tensors that will be fed during inference
    :param outputs: a list of tensorflow tensors that will be fetched during inference
    :return:
    """

    output_node_names = list({t.op.name for t in outputs})

    graph_def = sess.graph_def
    graph = sess.graph

    # clear device specifications
    for node in graph_def.node:
        node.device = ""

    non_placeholder_input_names = []
    type_enums = []
    for input_tensor in inputs:
        if input_tensor.op.type != "Placeholder":
            non_placeholder_input_names.append(input_tensor.name)
            type_enums.append(input_tensor.dtype.as_datatype_enum)

    output_names = [o.name for o in outputs]

    all_variables = graph.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)

    # freeze graph
    frozen_graph_def = graph_util.convert_variables_to_constants(
        sess,
        graph_def,
        output_node_names
    )

    optimized_graph_def, old_names2new = strip_unused(frozen_graph_def,
                                                      non_placeholder_input_names,
                                                      output_names,
                                                      type_enums)

    nodes_of_graph = []
    for node in optimized_graph_def.node:
        nodes_of_graph.append(node.name + ":0")
    nodes_of_graph_set = set(nodes_of_graph)

    new_input_names = []
    error_input_nodes = []
    for t in inputs:
        if t.name in old_names2new:
            if old_names2new[t.name] not in nodes_of_graph_set:
                error_input_nodes.append("\"" + (t.name)[0:-2] + "\"")
            new_input_names.append(old_names2new[t.name])
        else:
            if t.name not in nodes_of_graph_set:
                error_input_nodes.append("\"" + (t.name)[0:-2] + "\"")
            new_input_names.append(t.name)

    if error_input_nodes:
        error_nodes_name = " and ".join(error_input_nodes)
        raise ValueError("Node %s doesn't exist in the graph" % str(error_nodes_name))

    # check all placeholder in the graph are listed in the new_input_names:
    new_input_nodes = {name.split(":")[0] for name in new_input_names}
    for node in optimized_graph_def.node:
        if node.op == "Placeholder" and node.name not in new_input_nodes:
            raise ValueError(
                "Node %s is a Placeholder but not listed in inputs, inputs are %s"
                % (node.name, inputs))

    temp_tensors = None
    used_variables = []
    grad_variables = []
    grad_inputs = []
    if generate_backward:
        nodes = set([n.name for n in optimized_graph_def.node])
        for v in all_variables:
            if v.op.name in nodes:
                used_variables.append(v.name)

        with tf.Graph().as_default() as g:
            tf.import_graph_def(optimized_graph_def, name='')
            output_tensors = [g.get_tensor_by_name(x) for x in output_names]
            grad_output_placeholders = [tf.placeholder(dtype=x.dtype,
                                                       name=x.name.split(":")[0] + "_grad",
                                                       shape=x.shape) for x in output_tensors]

            variables = [g.get_tensor_by_name(x) for x in used_variables]

            inputs = [g.get_tensor_by_name(x) for x in new_input_names]
            grads = tf.gradients(output_tensors, variables + inputs,
                                 grad_ys=grad_output_placeholders)

            grads = [process_grad(grad) for grad in grads]

            temp_tensors = _find_temp_tensors(grads, nodes)

            grad_variables = [x.name for x in grads[0:len(variables)]]

            grad_inputs = []
            for i in range(len(variables), len(grads)):
                grad = grads[i]
                if grad is not None:
                    grad_inputs.append(grad.name)
                else:
                    # if input is not differentiable, we just return zero
                    input_tensor = inputs[i - len(variables)]
                    if allow_non_differentiable_input:
                        zero_grad = tf.zeros(shape=tf.shape(input_tensor))
                        grad_inputs.append(zero_grad.name)
                    else:
                        raise ValueError(
                            "input tensor: %s is not differentiable" % input_tensor.name)

            optimized_graph_def = g.as_graph_def()

    if not os.path.isdir(folder):
        os.makedirs(folder)

    with gfile.GFile(os.path.join(folder, "frozen_inference_graph.pb"), "wb") as f:
        f.write(optimized_graph_def.SerializeToString())

    meta = {
        "input_names": new_input_names,
        "output_names": output_names
    }

    if generate_backward:
        meta["temp_tensors"] = list(temp_tensors)
        meta["variables"] = used_variables
        meta["grad_variables"] = grad_variables
        meta["grad_inputs"] = grad_inputs

    with open(os.path.join(folder, "graph_meta.json"), "w") as f:
        f.write(json.dumps(meta))

def _find_temp_tensors(grads, forward_ops):
    '''
    find all the tensors that are used for computing grads and has been
    computed during forward
    :param grads:
    :param forward_ops:
    :return:
    '''
    import sys
    is_py2 = sys.version[0] == '2'
    if is_py2:
        import Queue as queue
    else:
        import queue as queue
    queue = queue.Queue()
    for grad in grads:
        queue.put(grad)

    temp_tensors = set()
    visited = set()
    while not queue.empty():
        tensor = queue.get()
        # this is necessary, because input may not be differentiable
        if tensor is None:
            continue
        else:
            visited.add(tensor.name)
            if tensor.op.type == "Placeholder":
                continue
            if tensor.op.name in forward_ops:
                temp_tensors.add(tensor.name)
                continue
            for input_tensor in tensor.op.inputs:
                # this is necessary because there may be a cycle in the graph such as tf.while_loop
                if input_tensor.name not in visited:
                    queue.put(input_tensor)
    return temp_tensors

def strip_unused(input_graph_def, input_tensor_names, output_tensor_names,
                 placeholder_type_enum):
    """Removes unused nodes from a GraphDef.

  Args:
    input_graph_def: A graph with nodes we want to prune.
    input_tensor_names: A list of the nodes we use as inputs.
    output_tensor_names: A list of the output nodes.
    placeholder_type_enum: The AttrValue enum for the placeholder data type, or
        a list that specifies one value per input node name.

  Returns:
    A `GraphDef` with all unnecessary ops removed. and a map containing the old input
    names to the new input names

  Raises:
    ValueError: If any element in `input_node_names` refers to a tensor instead
      of an operation.
    KeyError: If any element in `input_node_names` is not found in the graph.
  """
    for name in input_tensor_names:
        if ":" not in name:
            raise ValueError("Input '%s' appears to refer to a Operation, "
                             "not a Tensor." % name)

    old2new = {}

    # Here we replace the nodes we're going to override as inputs with
    # placeholders so that any unused nodes that are inputs to them are
    # automatically stripped out by extract_sub_graph().
    not_found = {name for name in input_tensor_names}
    input_node_names = {name.split(":")[0] for name in input_tensor_names}
    output_node_names = list({name.split(":")[0] for name in output_tensor_names})
    inputs_replaced_graph_def = graph_pb2.GraphDef()
    for node in input_graph_def.node:
        if node.name not in input_node_names:
            for i in range(len(node.input)):
                if _append_port(node.input[i]) in input_tensor_names:
                    old_name = _append_port(node.input[i])
                    not_found.remove(old_name)
                    new_input_name = node.input[i].replace(":", "_")
                    placeholder_node = node_def_pb2.NodeDef()
                    placeholder_node.op = "Placeholder"
                    placeholder_node.name = new_input_name
                    if isinstance(placeholder_type_enum, list):
                        input_node_index = input_tensor_names.index(old_name)
                        placeholder_node.attr["dtype"].CopyFrom(
                            attr_value_pb2.AttrValue(type=placeholder_type_enum[
                                input_node_index]))
                    else:
                        placeholder_node.attr["dtype"].CopyFrom(
                            attr_value_pb2.AttrValue(type=placeholder_type_enum))
                    if "_output_shapes" in node.attr:
                        placeholder_node.attr["_output_shapes"].CopyFrom(
                            node.attr["_output_shapes"])
                    node.input[i] = new_input_name
                    old2new[old_name] = new_input_name + ":0"
                    inputs_replaced_graph_def.node.extend([placeholder_node])
            inputs_replaced_graph_def.node.extend([copy.deepcopy(node)])

    if not_found:
        raise KeyError("The following input nodes were not found: %s\n" % not_found)

    output_graph_def = graph_util.extract_sub_graph(inputs_replaced_graph_def,
                                                    output_node_names)
    return output_graph_def, old2new

def _append_port(input_name):
    if input_name.find(":") == -1:
        return input_name + ":0"
    else:
        return input_name
<commit_msg>add keras learning phase support (#1065)<commit_after>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from tensorflow.core.framework import attr_value_pb2
from tensorflow.core.framework import graph_pb2
from tensorflow.core.framework import node_def_pb2
from tensorflow.python.framework import graph_util
from tensorflow.python.framework import ops
from tensorflow.python.ops import variable_scope
from tensorflow.python.platform import gfile
from tensorflow.python.util import tf_contextlib
import tensorflow as tf
import os
import json
import copy

def _variable_creator(next_creator, **kwargs):
    kwargs["use_resource"] = False
    return next_creator(**kwargs)

@tf_contextlib.contextmanager
def variable_creator_scope():
    with variable_scope.variable_creator_scope(_variable_creator):
        yield

def process_grad(grad):
    if grad is not None:
        grad = ops.convert_to_tensor_or_indexed_slices(grad)
        if isinstance(grad, ops.IndexedSlices):
            # In IndexedSlices is not supported in java api, we have to convert it to
            # a dense tensor. This operation is potentially expensive, but there seems
            # no work around
            grad = tf.unsorted_segment_sum(grad.values, grad.indices,
                                           grad.dense_shape[0])
    return grad

def export_tf(sess, folder, inputs, outputs,
              generate_backward=False, allow_non_differentiable_input=True):
    """
    Export the frozen tensorflow graph as well as the inputs/outputs information
    to the folder for inference.

    This function will
    1. freeze the graph (replace all variables with constants)
    2. strip all unused node as specified by inputs and outputs
    3. add placeholder nodes as needed
    4. write the frozen graph and inputs/outputs names to the folder

    Note: There should not be any queuing operation between inputs and outputs

    :param sess: tensorflow session holding the variables to be saved
    :param folder: the folder where graph file and inputs/outputs information are saved
    :param inputs: a list of tensorflow tensors that will be fed during inference
    :param outputs: a list of tensorflow tensors that will be fetched during inference
    :return:
    """

    output_node_names = list({t.op.name for t in outputs})

    graph_def = sess.graph_def
    graph = sess.graph

    # clear device specifications
    for node in graph_def.node:
        node.device = ""

    non_placeholder_input_names = []
    type_enums = []
    for input_tensor in inputs:
        if input_tensor.op.type not in ["Placeholder", "PlaceholderWithDefault"]:
            non_placeholder_input_names.append(input_tensor.name)
            type_enums.append(input_tensor.dtype.as_datatype_enum)

    output_names = [o.name for o in outputs]

    all_variables = graph.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)

    # freeze graph
    frozen_graph_def = graph_util.convert_variables_to_constants(
        sess,
        graph_def,
        output_node_names
    )

    optimized_graph_def, old_names2new = strip_unused(frozen_graph_def,
                                                      non_placeholder_input_names,
                                                      output_names,
                                                      type_enums)

    nodes_of_graph = []
    for node in optimized_graph_def.node:
        nodes_of_graph.append(node.name + ":0")
    nodes_of_graph_set = set(nodes_of_graph)

    new_input_names = []
    error_input_nodes = []
    for t in inputs:
        if t.name in old_names2new:
            if old_names2new[t.name] not in nodes_of_graph_set:
                error_input_nodes.append("\"" + (t.name)[0:-2] + "\"")
            new_input_names.append(old_names2new[t.name])
        else:
            if t.name not in nodes_of_graph_set:
                error_input_nodes.append("\"" + (t.name)[0:-2] + "\"")
            new_input_names.append(t.name)

    if error_input_nodes:
        error_nodes_name = " and ".join(error_input_nodes)
        raise ValueError("Node %s doesn't exist in the graph" % str(error_nodes_name))

    # check all placeholder in the graph are listed in the new_input_names:
    new_input_nodes = {name.split(":")[0] for name in new_input_names}
    for node in optimized_graph_def.node:
        if node.op == "Placeholder" and node.name not in new_input_nodes:
            raise ValueError(
                "Node %s is a Placeholder but not listed in inputs, inputs are %s"
                % (node.name, inputs))

    temp_tensors = None
    used_variables = []
    grad_variables = []
    grad_inputs = []
    if generate_backward:
        nodes = set([n.name for n in optimized_graph_def.node])
        for v in all_variables:
            if v.op.name in nodes:
                used_variables.append(v.name)

        with tf.Graph().as_default() as g:
            tf.import_graph_def(optimized_graph_def, name='')
            output_tensors = [g.get_tensor_by_name(x) for x in output_names]
            grad_output_placeholders = [tf.placeholder(dtype=x.dtype,
                                                       name=x.name.split(":")[0] + "_grad",
                                                       shape=x.shape) for x in output_tensors]

            variables = [g.get_tensor_by_name(x) for x in used_variables]

            inputs = [g.get_tensor_by_name(x) for x in new_input_names]
            grads = tf.gradients(output_tensors, variables + inputs,
                                 grad_ys=grad_output_placeholders)

            grads = [process_grad(grad) for grad in grads]

            temp_tensors = _find_temp_tensors(grads, nodes)

            grad_variables = [x.name for x in grads[0:len(variables)]]

            grad_inputs = []
            for i in range(len(variables), len(grads)):
                grad = grads[i]
                if grad is not None:
                    grad_inputs.append(grad.name)
                else:
                    # if input is not differentiable, we just return zero
                    input_tensor = inputs[i - len(variables)]
                    if allow_non_differentiable_input:
                        zero_grad = tf.zeros(shape=tf.shape(input_tensor))
                        grad_inputs.append(zero_grad.name)
                    else:
                        raise ValueError(
                            "input tensor: %s is not differentiable" % input_tensor.name)

            optimized_graph_def = g.as_graph_def()

    if not os.path.isdir(folder):
        os.makedirs(folder)

    with gfile.GFile(os.path.join(folder, "frozen_inference_graph.pb"), "wb") as f:
        f.write(optimized_graph_def.SerializeToString())

    meta = {
        "input_names": new_input_names,
        "output_names": output_names
    }

    if generate_backward:
        meta["temp_tensors"] = list(temp_tensors)
        meta["variables"] = used_variables
        meta["grad_variables"] = grad_variables
        meta["grad_inputs"] = grad_inputs

    with open(os.path.join(folder, "graph_meta.json"), "w") as f:
        f.write(json.dumps(meta))

def _find_temp_tensors(grads, forward_ops):
    '''
    find all the tensors that are used for computing grads and has been
    computed during forward
    :param grads:
    :param forward_ops:
    :return:
    '''
    import sys
    is_py2 = sys.version[0] == '2'
    if is_py2:
        import Queue as queue
    else:
        import queue as queue
    queue = queue.Queue()
    for grad in grads:
        queue.put(grad)

    temp_tensors = set()
    visited = set()
    while not queue.empty():
        tensor = queue.get()
        # this is necessary, because input may not be differentiable
        if tensor is None:
            continue
        else:
            visited.add(tensor.name)
            if tensor.op.type == "Placeholder":
                continue
            if tensor.op.name in forward_ops:
                temp_tensors.add(tensor.name)
                continue
            for input_tensor in tensor.op.inputs:
                # this is necessary because there may be a cycle in the graph such as tf.while_loop
                if input_tensor.name not in visited:
                    queue.put(input_tensor)
    return temp_tensors

def strip_unused(input_graph_def, input_tensor_names, output_tensor_names,
                 placeholder_type_enum):
    """Removes unused nodes from a GraphDef.

  Args:
    input_graph_def: A graph with nodes we want to prune.
    input_tensor_names: A list of the nodes we use as inputs.
    output_tensor_names: A list of the output nodes.
    placeholder_type_enum: The AttrValue enum for the placeholder data type, or
        a list that specifies one value per input node name.

  Returns:
    A `GraphDef` with all unnecessary ops removed. and a map containing the old input
    names to the new input names

  Raises:
    ValueError: If any element in `input_node_names` refers to a tensor instead
      of an operation.
    KeyError: If any element in `input_node_names` is not found in the graph.
  """
    for name in input_tensor_names:
        if ":" not in name:
            raise ValueError("Input '%s' appears to refer to a Operation, "
                             "not a Tensor." % name)

    old2new = {}

    # Here we replace the nodes we're going to override as inputs with
    # placeholders so that any unused nodes that are inputs to them are
    # automatically stripped out by extract_sub_graph().
    not_found = {name for name in input_tensor_names}
    input_node_names = {name.split(":")[0] for name in input_tensor_names}
    output_node_names = list({name.split(":")[0] for name in output_tensor_names})
    inputs_replaced_graph_def = graph_pb2.GraphDef()
    for node in input_graph_def.node:
        if node.name not in input_node_names:
            for i in range(len(node.input)):
                if _append_port(node.input[i]) in input_tensor_names:
                    old_name = _append_port(node.input[i])
                    not_found.remove(old_name)
                    new_input_name = node.input[i].replace(":", "_")
                    placeholder_node = node_def_pb2.NodeDef()
                    placeholder_node.op = "Placeholder"
                    placeholder_node.name = new_input_name
                    if isinstance(placeholder_type_enum, list):
                        input_node_index = input_tensor_names.index(old_name)
                        placeholder_node.attr["dtype"].CopyFrom(
                            attr_value_pb2.AttrValue(type=placeholder_type_enum[
                                input_node_index]))
                    else:
                        placeholder_node.attr["dtype"].CopyFrom(
                            attr_value_pb2.AttrValue(type=placeholder_type_enum))
                    if "_output_shapes" in node.attr:
                        placeholder_node.attr["_output_shapes"].CopyFrom(
                            node.attr["_output_shapes"])
                    node.input[i] = new_input_name
                    old2new[old_name] = new_input_name + ":0"
                    inputs_replaced_graph_def.node.extend([placeholder_node])
            inputs_replaced_graph_def.node.extend([copy.deepcopy(node)])

    if not_found:
        raise KeyError("The following input nodes were not found: %s\n" % not_found)

    output_graph_def = graph_util.extract_sub_graph(inputs_replaced_graph_def,
                                                    output_node_names)
    return output_graph_def, old2new

def _append_port(input_name):
    if input_name.find(":") == -1:
        return input_name + ":0"
    else:
        return input_name
<|endoftext|>


<commit_before>from urllib.request import Request
from urllib.request import urlopen
from urllib.error import HTTPError
from pymongo import MongoClient
from time import sleep
from urllib.parse import urlparse
from random import randrange

from os import path
from os import mkdir

from bs4 import BeautifulSoup

# 网站根路径
baseUrl = "http://www.ja7lib.com/cn"
# 新话题
urlUpdate = "http://www.ja7lib.com/cn/vl_update.php"
# 新发行-有评论
urlNewRelease = "http://www.ja7lib.com/cn/vl_newrelease.php?&mode=1"
# 新发行-全部
urlNewReleaseAll = "http://www.ja7lib.com/cn/vl_newrelease.php?&mode=2"
# 新加入
urlNewEntries = "http://www.ja7lib.com/cn/vl_newentries.php"
# 最想要-上个月
urlMostWanted = "http://www.ja7lib.com/cn/vl_mostwanted.php?&mode=1"
# 最想要-全部
urlMostWantedAll = "http://www.ja7lib.com/cn/vl_mostwanted.php?&mode=2"
# 高评价-上个月
urlBestrated = "http://www.ja7lib.com/cn/vl_bestrated.php?&mode=1"
# 高评价-全部
urlBestratedAll = "http://www.ja7lib.com/cn/vl_bestrated.php?&mode=2"

# 存储video未爬过链接
videoDic = {}
# 最大页
maxPage = 1
# 当前页
currentPage = 1

client = MongoClient("localhost", 27017)
db = client.jav

# 增加video链接
def get_video_urls(url):
    try:
        # 将html转换成Python对象
        bsObj = get_pagebs(url)
        # 找到所有video的div标签
        videoTags = bsObj.find_all("div", class_="video")
        for videoTag in videoTags:
            # 获取第一个a标签
            href = videoTag.a["href"]
            if href:
                # 获取链接绝对路径
                href = baseUrl + href[1:]
            # string为标签内的字符串
            videoNum = videoTag.find("div", class_="id").string
            if not videoNum in videoDic and not exist_video_db(videoNum):
                videoDic[videoNum] = href
        # 使用全局变量才能赋值
        global currentPage
        currentPage = currentPage + 1
        if currentPage > maxPage:
            return None
        bsNextPage = bsObj.find("a", class_="page next")
        if bsNextPage:
            nextPageUrl = baseUrl[:-3] + bsNextPage["href"]
            get_video_urls(nextPageUrl)
    except AttributeError as e:
        print(e)
        return None

# 获取页面beautifulsoup对象
def get_pagebs(url):
    html = url_open(url)
    # 将html转换成Python对象
    bsObj = BeautifulSoup(html.decode("utf8"), "html.parser")
    return bsObj

# 打开网址读取返回信息
def url_open(url):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Host": urlparse(url).netloc
    }
    # 请求添加heads
    req = Request(url, headers=headers)

    try:
        response = urlopen(req)
        return response.read()
    except HTTPError as e:
        print(e)
        return None

# 添加video信息
def get_video_infos():
    if videoDic.__len__() == 0:
        print("new page url can not be found")
        return None

    try:
        for url in videoDic.values():
            # 将html转换成Python对象
            bsObj = get_pagebs(url)
            video = {}
            video["title"] = bsObj.find(id="video_title").a.string
            video["imgUrl"] = bsObj.find(id="video_jacket_img")["src"]
            video["videoNum"] = bsObj.find(id="video_id").find("td", class_="text").string
            video["date"] = bsObj.find(id="video_date").find("td", class_="text").string
            video["length"] = bsObj.find(id="video_length").find("span", class_="text").string
            video["director"] = get_field_list(bsObj.find(id="video_director"))
            video["maker"] = get_field_list(bsObj.find(id="video_maker"))
            video["label"] = get_field_list(bsObj.find(id="video_label"))
            if bsObj.find(id="video_review"):
                video["review"] = bsObj.find(id="video_review").find("span", class_="score").string[1:-1]
            video["genres"] = get_field_list(bsObj.find(id="video_genres"))
            video["cast"] = get_field_list(bsObj.find(id="video_cast"))
            video["usersWanted"] = bsObj.find(id="subscribed").a.string
            video["usersWatched"] = bsObj.find(id="watched").a.string
            video["userOwned"] = bsObj.find(id="owned").a.string
            print(video)
            insert_video_db(video)

    except AttributeError as e:
        print(e)
        return None

# 获取字段列表
def get_field_list(bsObj):
    list = []
    if bsObj:
        tags = bsObj.find("td", class_="text").find_all("a")
        for tag in tags:
            name = tag.string
            url = baseUrl + "/" + tag["href"]
            list.append({"name": name, "url": url})
    return list

# 增加video信息到数据库中
def insert_video_db(video):
    if not exist_video_db(video["videoNum"]):
        return db.videos.insert_one(video)

def exist_video_db(videoNum):
    return db.videos.find_one({"videoNum": videoNum})

def find_download_videos_db():
    return db.videos.find({"status": {"$nin": ["downloaded"]}})

def update_video_status_db(video, status):
    db.videos.update_one({"videoNum": video["videoNum"]}, {"$set": {"status": status}})

# 下载图片
def download_imgs():
    videoInfoList = find_download_videos_db()
    if videoInfoList.count() == 0:
        print("new img url can not be found")
        return None

    # 图片保存路径
    save_path = path.abspath("./downlaods")
    if not path.exists(save_path):
        mkdir(save_path)

    print("download begin...")

    for video in videoInfoList:
        try:
            filename = video["videoNum"] + '.jpg'
            filePath = path.join(save_path, filename)
            if not path.exists(filePath):
                image = url_open(video["imgUrl"])
                with open(filePath, "wb") as f:
                    f.write(image)
                print("Done: ", filename)
            # 更新已下载状态
            update_video_status_db(video, "downloaded")
        except HTTPError as e:
            print(e)
            continue

    print("download end...")

if __name__ == "__main__":
    get_video_urls(urlBestrated)

    get_video_infos()

    download_imgs()
<commit_msg>support thread<commit_after>from urllib.request import Request
from urllib.request import urlopen
from urllib.error import HTTPError, URLError
from pymongo import MongoClient
from time import sleep
from urllib.parse import urlparse
from random import randrange
from threading import Lock, Thread
from queue import Queue
from atexit import register
from socket import timeout, setdefaulttimeout
import sys

from os import path
from os import mkdir

from bs4 import BeautifulSoup

# 网站根路径
baseUrl = "http://www.ja7lib.com/cn"
# 新话题
urlUpdate = baseUrl + "/vl_update.php"
# 新发行-有评论
urlNewRelease = baseUrl + "/vl_newrelease.php?&mode=1"
# 新发行-全部
urlNewReleaseAll = baseUrl + "/vl_newrelease.php?&mode=2"
# 新加入
urlNewEntries = baseUrl + "/vl_newentries.php"
# 最想要-上个月
urlMostWanted = baseUrl + "/vl_mostwanted.php?&mode=1"
# 最想要-全部
urlMostWantedAll = baseUrl + "/vl_mostwanted.php?&mode=2"
# 高评价-上个月
urlBestrated = baseUrl + "/vl_bestrated.php?&mode=1"
# 高评价-全部
urlBestratedAll = baseUrl + "/vl_bestrated.php?&mode=2"

# 最大页
maxPage = 5
# 当前页
currentPage = 1
# 是否完成
isFinish = 0
# 同步锁
lock = Lock()

client = MongoClient("localhost", 27017)
db = client.jav

# 这里对整个socket层设置超时时间
timeout = 30
setdefaulttimeout(timeout)

# 增加video链接
def get_video_urls(url, queue):
    try:
        # 将html转换成Python对象
        bsObj = get_pagebs(url)
        # 找到所有video的div标签
        videoTags = bsObj.find_all("div", class_="video")
        for videoTag in videoTags:
            try:
                # 获取第一个a标签
                href = videoTag.a["href"]
                # 获取链接绝对路径
                href = baseUrl + href[1:]
                # string为标签内的字符串
                videoNum = videoTag.find("div", class_="id").string
                # 在有可用空间之前阻塞
                queue.put({"videoNum": videoNum, "href": href}, 1)
            except Exception as e:
                print(str.format("Position:{0},Error:{1}", get_current_function_name(), e))
                continue
        print(url)
        # 使用global才能赋值全局变量
        global currentPage
        global isFinish
        currentPage = currentPage + 1
        if currentPage > maxPage:
            lock.acquire()
            isFinish = 1
            lock.release()
            print("get video urls done")
            return None
        bsNextPage = bsObj.find("a", class_="page next")
        if bsNextPage:
            nextPageUrl = baseUrl[:-3] + bsNextPage["href"]
            get_video_urls(nextPageUrl, queue)
    except Exception as e:
        print(str.format("Position:{0},Error:{1}", __name__, e))
        return None

# 获取当前函数名
def get_current_function_name():
    return sys._getframe(1).f_code.co_name

# 获取页面beautifulsoup对象
def get_pagebs(url):
    html = url_open(url)
    # 将html转换成Python对象
    bsObj = BeautifulSoup(html.decode("utf8"), "html.parser")
    return bsObj

# 打开网址读取返回信息
def url_open(url):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Host": urlparse(url).netloc
    }
    # 请求添加heads
    req = Request(url, headers=headers)

    try:
        request = urlopen(req)
        content = request.read()
        request.close()
        return content
    except HTTPError as e:
        print('-----HTTPError url:', e)
    except UnicodeDecodeError as e:
        print('-----UnicodeDecodeError url:', e)
    except URLError as e:
        print("-----urlError url:", e)
    except timeout as e:
        print("-----socket timout:", e)

# 添加video信息
def get_video_infos(queue):
    global isFinish
    while True:
        if isFinish == 1 and queue.empty():
            lock.acquire()
            isFinish = 2
            lock.release()
            print("get video infos done")
            return None
        try:
            val = queue.get(1)
            if not exist_video_db(val["videoNum"]):
                get_video_info(val["href"])
        except Exception as e:
            print(str.format("Position:{0},Error:{1}", get_current_function_name(), e))
            continue

def get_video_info(url):
    # 将html转换成Python对象
    bsObj = get_pagebs(url)
    video = {}
    video["title"] = bsObj.find(id="video_title").a.string
    video["imgUrl"] = bsObj.find(id="video_jacket_img")["src"]
    video["videoNum"] = bsObj.find(id="video_id").find("td", class_="text").string
    video["date"] = bsObj.find(id="video_date").find("td", class_="text").string
    video["length"] = bsObj.find(id="video_length").find("span", class_="text").string
    video["director"] = get_field_list(bsObj.find(id="video_director"))
    video["maker"] = get_field_list(bsObj.find(id="video_maker"))
    video["label"] = get_field_list(bsObj.find(id="video_label"))
    if bsObj.find(id="video_review"):
        video["review"] = bsObj.find(id="video_review").find("span", class_="score").string[1:-1]
    video["genres"] = get_field_list(bsObj.find(id="video_genres"))
    video["cast"] = get_field_list(bsObj.find(id="video_cast"))
    video["usersWanted"] = bsObj.find(id="subscribed").a.string
    video["usersWatched"] = bsObj.find(id="watched").a.string
    video["userOwned"] = bsObj.find(id="owned").a.string
    print(video)
    insert_video_db(video)

# 获取字段列表
def get_field_list(bsObj):
    list = []
    if bsObj:
        tags = bsObj.find("td", class_="text").find_all("a")
        for tag in tags:
            name = tag.string
            url = baseUrl + "/" + tag["href"]
            list.append({"name": name, "url": url})
    return list

# 增加video信息到数据库中
def insert_video_db(video):
    return db.videos.insert_one(video)

# 检查video是否存在数据库中
def exist_video_db(videoNum):
    return db.videos.find_one({"videoNum": videoNum})

# 下载图片
def download_imgs():
    # 图片保存路径
    save_path = path.abspath("./downlaods")
    if not path.exists(save_path):
        mkdir(save_path)

    while True:
        videoInfoList = find_download_videos_db()
        if isFinish == 2 and videoInfoList.count() == 0:
            print("download images done")
            return None
        for video in videoInfoList:
            download_img(video, save_path)
            sleep(randrange(1, 5))
        sleep(5)

def download_img(video, save_path):
    # 直到下载图片成功
    while True:
        try:
            filename = video["videoNum"] + '.jpg'
            filePath = path.join(save_path, filename)
            if not path.exists(filePath):
                image = url_open(video["imgUrl"])
                with open(filePath, "wb") as f:
                    f.write(image)
                print("Done: ", filename)
            # 更新已下载状态
            update_video_status_db(video, "downloaded")
            break
        except Exception as e:
            print(str.format("Position:{0},Error:{1}", get_current_function_name(), e))
            sleep(randrange(1, 5))

# 找到需要下载图片的video
def find_download_videos_db():
    return db.videos.find({"status": {"$nin": ["downloaded"]}})

# 更新video的状态已下载
def update_video_status_db(video, status):
    db.videos.update_one({"videoNum": video["videoNum"]}, {"$set": {"status": status}})

class MyThread(Thread):
    def __init__(self, func, args, name=""):
        Thread.__init__(self)
        self.name = name
        self.func = func
        self.args = args

    def getResult(self):
        return self.res

    # 重写父类run方法，在线程启动后执行该方法内的代码。
    def run(self):
        self.res = self.func(*self.args)

def main():
    # 存储video队列
    pageQ = Queue(100)
    funcs = [get_video_urls, get_video_infos, download_imgs]
    threads = []
    for func in funcs:
        args = ()
        if func.__name__ == "get_video_urls":
            args = (urlBestrated, pageQ)
        elif func.__name__ == "get_video_infos":
            args = (pageQ,)
        t = MyThread(func, args, func.__name__)
        threads.append(t)

    for t in threads:
        t.start()

    for t in threads:
        t.join()

# 退出函数，会在脚本退出之前请求调用这个特殊函数
@register
def _atexit():
    print("all done")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Sbud raw hex viewer

# Originally from HexII:
#  because ASCII dump is usually useless,
#  unless there is an ASCII string,
#  in which case the HEX part is useless.

# Ange Albertini, BSD Licence 2014-2020

# v0.14:
# - Python 3
# - ASCII is using a threshold - too many FPs
# - Zero display is using a threshold - too many FPs
# - no more alpha character
# - Hex ruler at the bottom
# - dots interleave line is displayed after a skip
# - gap size displayed
# - fixed ruler with different line length
# - ANSI colors, themes
# - charsets
# - filename and hash
# - arguments parsing
# - Ansi style optimisation
# - style propagation over nibbles
# - compact mode via alternating backgrounds

# v0.13:
# Hex:
# - ASCII chars are replaced as .<char>
#   (optionally in colors)
# - 00 is replaced by "  "
# - FF is replaced by "##"
# - other chars are returned as hex
#
# Output:
# - a hex ruler is shown at the top of the 'display'
# - offsets don't contain superfluous 0
# - offsets are removed identical starting characters if following the previous
# - lines full of 00 are skipped
# - offsets after a skip are fully written
# - Last_Offset+1 is marked with "]"
#   (because EOF could be absent)

# Todo
# - text output ?
# - clean compact mode: theme, ascii
# - ASCII coloring bugged (cf WAD)
# - alpha coloring lost with compact mode (cf WAD)

import hashlib
import argparse
import sys
from string import punctuation, digits, ascii_letters
import ansi
import themes

def propStyles(b):
    # propagate styles from one nibble to the next if identical
    raw, fgs, bgs = ansi.getStyles(b)

    for i,c in enumerate(raw):
        if c == ord(" ") and i in fgs and i+1 in fgs:
            del(fgs[i])
        if c == ord(" ") and i in bgs and i+1 in bgs:
            del(bgs[i])

    result = ansi.generate(raw, fgs, bgs)
    return result

def setAltBgs(b, bg2):
    """alternate backgrounds for each nibble"""
    global bCompact
    if not bCompact:
        return b

    bIsStr = False
    if isinstance(b, str):
        bIsStr = True
        b = b.encode("utf-8")
    raw, fgs, bgs = ansi.getStyles(b)
    bg = ansi.Colors.ResetBG
    fg = ansi.Colors.ResetFG
    for i in range(len(raw)):
        if i in bgs:
            bg = bgs[i]
        if i in fgs:
            fg = fgs[i]
        if bg != ansi.Colors.ResetBG:
            continue
        if i % 4 == 0:
            bgs[i] = ansi.Colors.ResetBG
        elif i % 4 == 2:

            # no need of alternate bg if we're showing an empty space
            if raw[i:i+2] == b"  ":
                continue

            # no need of alternate bg if we're surrounded by empty space
            emptyBef = False
            if i < 2:
                emptyBef = True
            elif raw[i-2:i] == b"  ":
                emptyBef = True

            emptyAft = False
            if i >= len(raw) - 2:
                emptyAft = True
            elif raw[i+2:i+4] == b"  ":
                emptyAft = True

            if emptyAft and emptyBef:
                continue

            if ansi.sameColor(fg, bg2):
                fgs[i] = ansi.switchInt(bg2 - 10)
                if i+2 not in fgs:
                    fgs[i+2] = fg

            bgs[i] = bg2
    result = ansi.generate(raw, fgs, bgs)
    if bIsStr:
        result = result.decode("utf-8")

    return result

class charset:
    end = "]]"
    skip = "-"
    skOff = ">"
    digits = ["%X" % i for i in range(16)]
    numbers = ["%X" % i for i in range(256)]

class csAscii(charset):
    pass

chrs = lambda start, end: [chr(i) for i in range(start, end)]
fully_circled_digits = sum([
    ["\u24ea"], # 0
    chrs(0x2460, 0x2469), # 1-9
    chrs(0x2469, 0x246F), # 11-15
    # chrs(0x24b6, 0x24bb), # A-F
    # chrs(0x24d0, 0x24d5), # a-z
    ], [])

neg_circled_digits = sum([
    ["\u24ff"], # 0
    chrs(0x2776, 0x277f),   # 1-9
    chrs(0x1f150, 0x1f155), # A-F
    ], [])

neg_circled_sserif = sum([
    ["\U0001f10c"], # 0
    chrs(0x278A, 0x2792), # 1-9
    ], [])

class csUnicode(charset):
    skip = "\u2508"
    # \u2219 Bullet Operator
    # \u2500 Box Drawings Light Horizontal
    # \u2508 Box Drawings Light Quadruple Dash Horizontal
    skOff = "\u254c"

charsets = {
    "ascii": csAscii,
    "unicode": csUnicode
}

bAlpha = 0
parser = argparse.ArgumentParser(description="Sbud raw hex viewer.")
parser.add_argument('file',
    help="input file(s).")
parser.add_argument('-t', '--theme', default="Dark",
    help="color theme: %s." % ", ".join(sorted(themes.themes)))
parser.add_argument('-c', '--charset', default="Unicode",
    help="charset: %s." % ", ".join(sorted(charsets)))

parser.add_argument('--compact', action="store_true",
    help="compact view mode.")

parser.add_argument('--row_length', type=int, default=16,
    help="row length.")
parser.add_argument('--zero_count', default=3, type=int,
    help="how many zeros bytes in a row.")
parser.add_argument('--ascii_count', default=3, type=int,
    help="how many Ascii bytes in a row.")

args = parser.parse_args()
theme = args.theme.lower()
charset = args.charset.lower()
bCompact = args.compact
ZeroT = args.zero_count
AsciiT = args.ascii_count
LINELEN = args.row_length

fn = args.file
with open(fn, "rb") as f:
    r = f.read()

ASCII = punctuation + digits + ascii_letters + " \n\r\x1a"
ASCII = ASCII.encode()

if theme not in themes.themes:
    print("Error: unknown theme %s, aborting." % repr(theme))
    sys.exit()
theme = themes.themes[theme]
if charset not in charsets:
    print("Error: unknown charset %s, aborting." % repr(charset))
    sys.exit()
charset = charsets[charset]

CHARS_NEEDED = len("%x" % len(r))

PREFIX = b"%%0%iX " % CHARS_NEEDED

#this should always be displayed on the top of the screen no matter the scrolling
skip_l = 3 if not bCompact else 2
joiner = " " if not bCompact else ""

numbers = joiner.join(charset.numbers[i].rjust(2) for i in range(LINELEN))
#print (numbers)
HeaderS = " " * CHARS_NEEDED + " " + ansi.marker(92) + setAltBgs(numbers, 42) + theme.reset

sha256 = hashlib.sha256(r).hexdigest()

print(theme.zero + "%s - %s..%s" % (repr(fn), sha256[:4], sha256[-4:])  + theme.reset)
print()
print(HeaderS)
print("")

# the first offset on top of a window should be completely displayed
last_off = None

def subst(r, i):
    global bAlpha
    c = r[i]
    ma = 0 if i == 0 else i

    #replace 00 by empty char
    if c == 0:
        count = 1
        mi = max(i-ZeroT, 0)
        bef, aft = r[mi:ma][::-1], r[i+1:i+ZeroT]
        for ss in [bef, aft]:
            for cc in ss:
                if cc != 0:
                    break
                count += 1

        if count >= ZeroT:
            bAlpha = 1 - bAlpha
            return "  "

    if c in ASCII:
        count = 1
        mi = max(i-AsciiT, 0)
        bef, aft = r[mi:ma][::-1], r[i+1:i+AsciiT]
        for ss in [bef, aft]:
            for cc in ss:
                if not cc in ASCII:
                    break
                count += 1

        if count >= AsciiT:
            if c == ord(" "):
                return theme.alpha[bAlpha] + "__" + theme.reset
            if c == ord("\n"):
                return theme.alpha[bAlpha] + "\\n" + theme.reset
            if c == ord("\r"):
                return theme.alpha[bAlpha] + "\\r" + theme.reset
            if c == 0x1a: # TODO: only in manual mode
                return theme.alpha[bAlpha] + "^Z" + theme.reset
            return theme.alpha[bAlpha] + " " + chr(c) + theme.reset
    bAlpha = 1 - bAlpha
    if c == 0:
        return theme.zero + "00" + theme.reset
    #otherwise, return hex
    return "%02X" % c

def csplit(s, n):
    for i in range(0, len(s), n):
        yield s[i:i+n]

l = []
for i in range(len(r)):
    l += [subst(r, i).encode("utf-8")]

l += [(theme.end + charset.end + theme.reset).encode("utf-8")]

skipping = False
before_skip = 0
for i, seq in enumerate(csplit(l, LINELEN)):
    l = b" ".join(seq) if not bCompact else b"".join(seq)

    if l.strip() != b"": #
        if skipping == True:
            gap_s = i * LINELEN - before_skip
            gap_str = " +%X" % gap_s
            gap_strl = len(gap_str)
            print(theme.skip
             + charset.skOff * (CHARS_NEEDED)
             + " "
             + charset.skip * (LINELEN) * skip_l
             + gap_str
             + theme.reset
             )
        skipping = False

        prefix = list(PREFIX % (i * LINELEN))

        #we'll skip starting chars if they are redundant
        if last_off is not None:
            save = last_off
            last_off = bytes(prefix)
            for i, j in enumerate(save):
                if prefix[i] == j:
                    prefix[i] = ord(" ")
                else:
                    break
        else:
            last_off = bytes(prefix)

        prefix = bytes(prefix)
        l = propStyles(l)
        l = setAltBgs(l, 100)
        print("%s%s" % (theme.offset+prefix.decode("utf-8")+theme.reset, l.decode("utf-8")))
    else:
        if skipping == False:
            before_skip = i * LINELEN
            skipping = True
            last_off = None
print("")
print(HeaderS)
<commit_msg>correct alpha alternating<commit_after>#!/usr/bin/env python3

# Sbud raw hex viewer

# Originally from HexII:
#  because ASCII dump is usually useless,
#  unless there is an ASCII string,
#  in which case the HEX part is useless.

# Ange Albertini, BSD Licence 2014-2020

# v0.14:
# - Python 3
# - ASCII is using a threshold - too many FPs
# - Zero display is using a threshold - too many FPs
# - no more alpha character
# - Hex ruler at the bottom
# - dots interleave line is displayed after a skip
# - gap size displayed
# - fixed ruler with different line length
# - ANSI colors, themes
# - charsets
# - filename and hash
# - arguments parsing
# - Ansi style optimisation
# - style propagation over nibbles
# - compact mode via alternating backgrounds, only when needed

# v0.13:
# Hex:
# - ASCII chars are replaced as .<char>
#   (optionally in colors)
# - 00 is replaced by "  "
# - FF is replaced by "##"
# - other chars are returned as hex
#
# Output:
# - a hex ruler is shown at the top of the 'display'
# - offsets don't contain superfluous 0
# - offsets are removed identical starting characters if following the previous
# - lines full of 00 are skipped
# - offsets after a skip are fully written
# - Last_Offset+1 is marked with "]"
#   (because EOF could be absent)

# Todo
# - text output ?
# - clean compact mode: theme, ascii
# - ASCII coloring bugged (cf WAD)
# - alpha coloring lost with compact mode (cf WAD)

import hashlib
import argparse
import sys
from string import punctuation, digits, ascii_letters
import ansi
import themes

def propStyles(b):
    # propagate styles from one nibble to the next if identical
    raw, fgs, bgs = ansi.getStyles(b)

    for i,c in enumerate(raw):
        if c == ord(" ") and i in fgs and i+1 in fgs:
            del(fgs[i])
        if c == ord(" ") and i in bgs and i+1 in bgs:
            del(bgs[i])

    result = ansi.generate(raw, fgs, bgs)
    return result

def setAltBgs(b, bg2):
    """alternate backgrounds for each nibble"""
    global bCompact
    if not bCompact:
        return b

    bIsStr = False
    if isinstance(b, str):
        bIsStr = True
        b = b.encode("utf-8")
    raw, fgs, bgs = ansi.getStyles(b)
    bg = ansi.Colors.ResetBG
    fg = ansi.Colors.ResetFG
    for i in range(len(raw)):
        if i in bgs:
            bg = bgs[i]
        if i in fgs:
            fg = fgs[i]
        if bg != ansi.Colors.ResetBG:
            continue
        if i % 4 == 0:
            bgs[i] = ansi.Colors.ResetBG
        elif i % 4 == 2:

            # no need of alternate bg if we're showing an empty space
            if raw[i:i+2] == b"  ":
                continue

            # no need of alternate bg if we're surrounded by empty space
            emptyBef = False
            if i < 2:
                emptyBef = True
            elif raw[i-2:i] == b"  " or i-2 in bgs:
                emptyBef = True

            emptyAft = False
            if i >= len(raw) - 2:
                emptyAft = True
            elif raw[i+2:i+4] == b"  " or i+2 in bgs:
                emptyAft = True

            if emptyAft and emptyBef:
                continue

            if ansi.sameColor(fg, bg2):
                fgs[i] = ansi.switchInt(bg2 - 10)
                if i+2 not in fgs:
                    fgs[i+2] = fg

            bgs[i] = bg2
    result = ansi.generate(raw, fgs, bgs)
    if bIsStr:
        result = result.decode("utf-8")

    return result

class charset:
    end = "]]"
    skip = "-"
    skOff = ">"
    digits = ["%X" % i for i in range(16)]
    numbers = ["%X" % i for i in range(256)]

class csAscii(charset):
    pass

chrs = lambda start, end: [chr(i) for i in range(start, end)]
fully_circled_digits = sum([
    ["\u24ea"], # 0
    chrs(0x2460, 0x2469), # 1-9
    chrs(0x2469, 0x246F), # 11-15
    # chrs(0x24b6, 0x24bb), # A-F
    # chrs(0x24d0, 0x24d5), # a-z
    ], [])

neg_circled_digits = sum([
    ["\u24ff"], # 0
    chrs(0x2776, 0x277f),   # 1-9
    chrs(0x1f150, 0x1f155), # A-F
    ], [])

neg_circled_sserif = sum([
    ["\U0001f10c"], # 0
    chrs(0x278A, 0x2792), # 1-9
    ], [])

class csUnicode(charset):
    skip = "\u2508"
    # \u2219 Bullet Operator
    # \u2500 Box Drawings Light Horizontal
    # \u2508 Box Drawings Light Quadruple Dash Horizontal
    skOff = "\u254c"

charsets = {
    "ascii": csAscii,
    "unicode": csUnicode
}

bAlpha = 0
parser = argparse.ArgumentParser(description="Sbud raw hex viewer.")
parser.add_argument('file',
    help="input file(s).")
parser.add_argument('-t', '--theme', default="Dark",
    help="color theme: %s." % ", ".join(sorted(themes.themes)))
parser.add_argument('-c', '--charset', default="Unicode",
    help="charset: %s." % ", ".join(sorted(charsets)))

parser.add_argument('--compact', action="store_true",
    help="compact view mode.")

parser.add_argument('--row_length', type=int, default=16,
    help="row length.")
parser.add_argument('--zero_count', default=3, type=int,
    help="how many zeros bytes in a row.")
parser.add_argument('--ascii_count', default=3, type=int,
    help="how many Ascii bytes in a row.")

args = parser.parse_args()
theme = args.theme.lower()
charset = args.charset.lower()
bCompact = args.compact
ZeroT = args.zero_count
AsciiT = args.ascii_count
LINELEN = args.row_length

fn = args.file
with open(fn, "rb") as f:
    r = f.read()

ASCII = punctuation + digits + ascii_letters + " \n\r\x1a"
ASCII = ASCII.encode()

if theme not in themes.themes:
    print("Error: unknown theme %s, aborting." % repr(theme))
    sys.exit()
theme = themes.themes[theme]
if charset not in charsets:
    print("Error: unknown charset %s, aborting." % repr(charset))
    sys.exit()
charset = charsets[charset]

CHARS_NEEDED = len("%x" % len(r))

PREFIX = b"%%0%iX " % CHARS_NEEDED

#this should always be displayed on the top of the screen no matter the scrolling
skip_l = 3 if not bCompact else 2
joiner = " " if not bCompact else ""

numbers = joiner.join(charset.numbers[i].rjust(2) for i in range(LINELEN))
#print (numbers)
HeaderS = " " * CHARS_NEEDED + " " + ansi.marker(92) + setAltBgs(numbers, 42) + theme.reset

sha256 = hashlib.sha256(r).hexdigest()

print(theme.zero + "%s - %s..%s" % (repr(fn), sha256[:4], sha256[-4:])  + theme.reset)
print()
print(HeaderS)
print("")

# the first offset on top of a window should be completely displayed
last_off = None

isAlpha = None

def leaveAlpha():
    global bAlpha, isAlpha
    if isAlpha == True:
        bAlpha = 1 - bAlpha
    isAlpha = False

def subst(r, i):
    global bAlpha, isAlpha
    c = r[i]
    ma = 0 if i == 0 else i

    #replace 00 by empty char
    if c == 0:
        count = 1
        mi = max(i-ZeroT, 0)
        bef, aft = r[mi:ma][::-1], r[i+1:i+ZeroT]
        for ss in [bef, aft]:
            for cc in ss:
                if cc != 0:
                    break
                count += 1

        if count >= ZeroT:
            leaveAlpha()
            return "  "

    if c in ASCII:
        count = 1
        mi = max(i-AsciiT, 0)
        bef, aft = r[mi:ma][::-1], r[i+1:i+AsciiT]
        for ss in [bef, aft]:
            for cc in ss:
                if not cc in ASCII:
                    break
                count += 1

        if count >= AsciiT:
            if c == ord(" "):
                isAlpha = True
                return theme.alpha[bAlpha] + "__" + theme.reset
            if c == ord("\n"):
                isAlpha = True
                return theme.alpha[bAlpha] + "\\n" + theme.reset
            if c == ord("\r"):
                isAlpha = True
                return theme.alpha[bAlpha] + "\\r" + theme.reset
            if c == 0x1a: # TODO: only in manual mode
                isAlpha = True
                return theme.alpha[bAlpha] + "^Z" + theme.reset

            isAlpha = True
            return theme.alpha[bAlpha] + " " + chr(c) + theme.reset
    leaveAlpha()

    if c == 0:
        return theme.zero + "00" + theme.reset
    #otherwise, return hex
    return "%02X" % c

def csplit(s, n):
    for i in range(0, len(s), n):
        yield s[i:i+n]

l = []
for i in range(len(r)):
    l += [subst(r, i).encode("utf-8")]

l += [(theme.end + charset.end + theme.reset).encode("utf-8")]

skipping = False
before_skip = 0
for i, seq in enumerate(csplit(l, LINELEN)):
    l = b" ".join(seq) if not bCompact else b"".join(seq)

    if l.strip() != b"": #
        if skipping == True:
            gap_s = i * LINELEN - before_skip
            gap_str = " +%X" % gap_s
            gap_strl = len(gap_str)
            print(theme.skip
             + charset.skOff * (CHARS_NEEDED)
             + " "
             + charset.skip * (LINELEN) * skip_l
             + gap_str
             + theme.reset
             )
        skipping = False

        prefix = list(PREFIX % (i * LINELEN))

        #we'll skip starting chars if they are redundant
        if last_off is not None:
            save = last_off
            last_off = bytes(prefix)
            for i, j in enumerate(save):
                if prefix[i] == j:
                    prefix[i] = ord(" ")
                else:
                    break
        else:
            last_off = bytes(prefix)

        prefix = bytes(prefix)
        l = propStyles(l)
        l = setAltBgs(l, 100)
        print("%s%s" % (theme.offset+prefix.decode("utf-8")+theme.reset, l.decode("utf-8")))
    else:
        if skipping == False:
            before_skip = i * LINELEN
            skipping = True
            last_off = None
print("")
print(HeaderS)
<|endoftext|>


<commit_before>import re
import collections
import functools

from django.core.exceptions import FieldError
from django.utils.six import with_metaclass
from django.db import models

_FLOAT_RE = r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?'

def require_postgres(fn):
    """
    Decorator that checks if the target backend engine is a PostgreSQL instance

    :raises: FieldError
    """

    def wrapper(self, connection):
        engine = connection.settings_dict['ENGINE']

        if 'psycopg2' not in engine and 'postgis' not in engine:
            raise FieldError("Current database is not a PostgreSQL instance")

        return fn(self, connection)

    return wrapper

@functools.total_ordering
class Point(object):
    """
    Describe a point in the space.
    """

    POINT_RE = r'\((?P<x>{0}),(?P<y>{0})\)'.format(_FLOAT_RE)

    @staticmethod
    def from_string(value):
        """
        Convert a string describing a point into a `Point` instance.

        The representation of a point as a string:

            (x, y)

        where `x` and `y` can be signed or unsigned integers or floats

        :param value: The string representation of the point
        :rtype: Point
        :raise: ValueError if the given string is not a valid point's
                representation
        """
        match = re.match(Point.POINT_RE, value)

        if not match:
            raise ValueError("Value {} is not a valid point".format(value))

        values = match.groupdict()

        return Point(float(values['x']), float(values['y']))

    def __init__(self, x=0, y=0):
        self.x = x
        self.y = y

    def __repr__(self):
        return '<Point({0.x},{0.y})>'.format(self)

    def __str__(self):
        return '({0.x},{0.y})'.format(self)

    def __unicode__(self):
        return unicode(self.__str__())

    def __eq__(self, other):
        return (isinstance(other, self.__class__)
                and self.x == other.x
                and self.y == other.y)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __lt__(self, other):
        return (isinstance(other, self.__class__)
                and self.x <= other.x
                and self.y <= other.y)

class Circle(object):
    """
    Describe a circle with center and radius
    """

    CIRCLE_RE = r'<{0},\s(?P<r>{1})>'.format(Point.POINT_RE, _FLOAT_RE)

    @staticmethod
    def from_string(value):
        """
        Convert a string describing a circle into a `Circle` instance.

        The representation of a circle as a string:

            <(x, y), r>

        where `x`, 'y' and `r` can be signed or unsigned integers or floats; 'x'
        and 'y' defines the center of the circle and 'r' the radius.

        :param value: The string representation of the circle
        :rtype: Circle
        :raise: ValueError if the given string is not a valid circle's
                representation
        """
        match = re.match(Circle.CIRCLE_RE, value)

        if not match:
            raise ValueError("Value {} is not a valid circle".format(value))

        values = match.groupdict()

        return Circle(
            float(values['x']), float(values['y']), float(values['r']))

    def __init__(self, *args):
        """
        Constructor accept up to 3 arguments with the following signatures:

        * `Circle(r)` creates a circle in the origin with radius r`
        * `Circle(<Point>, r)` creates a circle with center in the given point
          and radius `r`
        * `Circle(x, y, r)` creates a circle with center in the given `x` and
          `y` coordinates and radius `r`
        """
        argc = len(args)

        if argc == 1:
            self.center = Point()
            self.radius = args[0]

        elif argc == 2 and isinstance(args[0], Point):
            self.center = args[0]
            self.radius = args[1]

        elif argc == 3:
            self.center = Point(*args[:2])
            self.radius = args[2]

        else:
            raise TypeError("Invalid set of arguments {}".format(args))

    def __eq__(self, other):
        return (isinstance(other, Circle)
                and self.center == other.center
                and self.radius == other.radius)

class PointMixin(object):

    SPLIT_RE = re.compile(r"\((?!\().*?\)")

    def to_python(self, values):
        if values is None:
            return None

        if not isinstance(values, collections.Iterable):
            raise TypeError("Value {} is not iterable".format(values))

        if all(isinstance(v, Point) for v in values):
            return values

        return list(
            Point.from_string(v) for v in re.findall(self.SPLIT_RE, values))

    def _get_prep_value(self, values):
        return ','.join(str(v) for v in values) if values else None

class SegmentPathField(PointMixin,
                       with_metaclass(models.SubfieldBase, models.Field)):
    """
    Field to store a path; needs at least two set of points
    """

    @require_postgres
    def db_type(self, connection):
        return 'path'

    def get_prep_value(self, values):
        if values:
            values = tuple(values)

            if len(values) < 2:
                raise ValueError("Needs at minimum 2 points")

        values = self._get_prep_value(values)

        return '[{}]'.format(values) if values else None

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

class PolygonField(PointMixin,
                   with_metaclass(models.SubfieldBase, models.Field)):
    """
    Field to store a polygon; needs at least three set of points
    """

    @require_postgres
    def db_type(self, connection):
        return 'polygon'

    def get_prep_value(self, values):
        if values:
            values = tuple(values)

            if len(values) < 3:
                raise ValueError("Needs at minimum 3 points")

        values = self._get_prep_value(values)

        return '({})'.format(values) if values else None

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

class PointField(with_metaclass(models.SubfieldBase, models.Field)):
    """
    Field to store a single point in space
    """

    @require_postgres
    def db_type(self, connection):
        return 'point'

    def to_python(self, value):
        if isinstance(value, Point) or value is None:
            return value

        return Point.from_string(value)

    def get_prep_value(self, value):
        return '({0.x},{0.y})'.format(value) if value else None

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

class SegmentField(PointMixin,
                   with_metaclass(models.SubfieldBase, models.Field)):
    """
    Field to store a path; needs exactly two set of points
    """

    @require_postgres
    def db_type(self, connection):
        return 'lseg'

    def get_prep_value(self, value):
        if value and len(value) != 2:
            raise ValueError("Segment needs exactly 2 points")

        return self._get_prep_value(value)

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

class BoxField(PointMixin, with_metaclass(models.SubfieldBase, models.Field)):
    """
    Field to store a box's definition.

    Needs two set of points defining the opposite corners of box. Any pair of
    opposite corners can be set to this field but on retrieve the upper-right
    and lower-left corners will be given back in this order.
    """

    @require_postgres
    def db_type(self, connection):
        return 'box'

    def get_prep_value(self, value):
        if value and len(value) != 2:
            raise ValueError("Box needs exactly 2 points")

        return self._get_prep_value(value)

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

class CircleField(with_metaclass(models.SubfieldBase, models.Field)):
    """
    Field to store a circle's definition
    """

    @require_postgres
    def db_type(self, connection):
        return 'circle'

    def to_python(self, value):
        if value is None or isinstance(value, Circle):
            return value

        return Circle.from_string(value)

    def get_prep_value(self, value):
        if value:
            return "<{0.center.x}, {0.center.y}}, {0.radius}>".format(value)

        return None

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

# Try to load South's model inspector
try:
    from south.modelsinspector import add_introspection_rules
    add_introspection_rules([], ['^postgres_geometry\.fields\.\w+Field'])

except ImportError:
    pass
<commit_msg>Remove broken use of with_metaclass<commit_after>import re
import collections
import functools

from django.core.exceptions import FieldError
from django.db import models

_FLOAT_RE = r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?'

def require_postgres(fn):
    """
    Decorator that checks if the target backend engine is a PostgreSQL instance

    :raises: FieldError
    """

    def wrapper(self, connection):
        engine = connection.settings_dict['ENGINE']

        if 'psycopg2' not in engine and 'postgis' not in engine:
            raise FieldError("Current database is not a PostgreSQL instance")

        return fn(self, connection)

    return wrapper

@functools.total_ordering
class Point(object):
    """
    Describe a point in the space.
    """

    POINT_RE = r'\((?P<x>{0}),(?P<y>{0})\)'.format(_FLOAT_RE)

    @staticmethod
    def from_string(value):
        """
        Convert a string describing a point into a `Point` instance.

        The representation of a point as a string:

            (x, y)

        where `x` and `y` can be signed or unsigned integers or floats

        :param value: The string representation of the point
        :rtype: Point
        :raise: ValueError if the given string is not a valid point's
                representation
        """
        match = re.match(Point.POINT_RE, value)

        if not match:
            raise ValueError("Value {} is not a valid point".format(value))

        values = match.groupdict()

        return Point(float(values['x']), float(values['y']))

    def __init__(self, x=0, y=0):
        self.x = x
        self.y = y

    def __repr__(self):
        return '<Point({0.x},{0.y})>'.format(self)

    def __str__(self):
        return '({0.x},{0.y})'.format(self)

    def __unicode__(self):
        return unicode(self.__str__())

    def __eq__(self, other):
        return (isinstance(other, self.__class__)
                and self.x == other.x
                and self.y == other.y)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __lt__(self, other):
        return (isinstance(other, self.__class__)
                and self.x <= other.x
                and self.y <= other.y)

class Circle(object):
    """
    Describe a circle with center and radius
    """

    CIRCLE_RE = r'<{0},\s(?P<r>{1})>'.format(Point.POINT_RE, _FLOAT_RE)

    @staticmethod
    def from_string(value):
        """
        Convert a string describing a circle into a `Circle` instance.

        The representation of a circle as a string:

            <(x, y), r>

        where `x`, 'y' and `r` can be signed or unsigned integers or floats; 'x'
        and 'y' defines the center of the circle and 'r' the radius.

        :param value: The string representation of the circle
        :rtype: Circle
        :raise: ValueError if the given string is not a valid circle's
                representation
        """
        match = re.match(Circle.CIRCLE_RE, value)

        if not match:
            raise ValueError("Value {} is not a valid circle".format(value))

        values = match.groupdict()

        return Circle(
            float(values['x']), float(values['y']), float(values['r']))

    def __init__(self, *args):
        """
        Constructor accept up to 3 arguments with the following signatures:

        * `Circle(r)` creates a circle in the origin with radius r`
        * `Circle(<Point>, r)` creates a circle with center in the given point
          and radius `r`
        * `Circle(x, y, r)` creates a circle with center in the given `x` and
          `y` coordinates and radius `r`
        """
        argc = len(args)

        if argc == 1:
            self.center = Point()
            self.radius = args[0]

        elif argc == 2 and isinstance(args[0], Point):
            self.center = args[0]
            self.radius = args[1]

        elif argc == 3:
            self.center = Point(*args[:2])
            self.radius = args[2]

        else:
            raise TypeError("Invalid set of arguments {}".format(args))

    def __eq__(self, other):
        return (isinstance(other, Circle)
                and self.center == other.center
                and self.radius == other.radius)

class PointMixin(object):

    SPLIT_RE = re.compile(r"\((?!\().*?\)")

    def to_python(self, values):
        if values is None:
            return None

        if not isinstance(values, collections.Iterable):
            raise TypeError("Value {} is not iterable".format(values))

        if all(isinstance(v, Point) for v in values):
            return values

        return list(
            Point.from_string(v) for v in re.findall(self.SPLIT_RE, values))

    def _get_prep_value(self, values):
        return ','.join(str(v) for v in values) if values else None

class SegmentPathField(PointMixin, models.Field):
    """
    Field to store a path; needs at least two set of points
    """

    @require_postgres
    def db_type(self, connection):
        return 'path'

    def get_prep_value(self, values):
        if values:
            values = tuple(values)

            if len(values) < 2:
                raise ValueError("Needs at minimum 2 points")

        values = self._get_prep_value(values)

        return '[{}]'.format(values) if values else None

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

class PolygonField(PointMixin, models.Field):
    """
    Field to store a polygon; needs at least three set of points
    """

    @require_postgres
    def db_type(self, connection):
        return 'polygon'

    def get_prep_value(self, values):
        if values:
            values = tuple(values)

            if len(values) < 3:
                raise ValueError("Needs at minimum 3 points")

        values = self._get_prep_value(values)

        return '({})'.format(values) if values else None

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

class PointField(models.Field):
    """
    Field to store a single point in space
    """

    @require_postgres
    def db_type(self, connection):
        return 'point'

    def to_python(self, value):
        if isinstance(value, Point) or value is None:
            return value

        return Point.from_string(value)

    def get_prep_value(self, value):
        return '({0.x},{0.y})'.format(value) if value else None

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

class SegmentField(PointMixin, models.Field):
    """
    Field to store a path; needs exactly two set of points
    """

    @require_postgres
    def db_type(self, connection):
        return 'lseg'

    def get_prep_value(self, value):
        if value and len(value) != 2:
            raise ValueError("Segment needs exactly 2 points")

        return self._get_prep_value(value)

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

class BoxField(PointMixin, models.Field):
    """
    Field to store a box's definition.

    Needs two set of points defining the opposite corners of box. Any pair of
    opposite corners can be set to this field but on retrieve the upper-right
    and lower-left corners will be given back in this order.
    """

    @require_postgres
    def db_type(self, connection):
        return 'box'

    def get_prep_value(self, value):
        if value and len(value) != 2:
            raise ValueError("Box needs exactly 2 points")

        return self._get_prep_value(value)

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

class CircleField(models.Field):
    """
    Field to store a circle's definition
    """

    @require_postgres
    def db_type(self, connection):
        return 'circle'

    def to_python(self, value):
        if value is None or isinstance(value, Circle):
            return value

        return Circle.from_string(value)

    def get_prep_value(self, value):
        if value:
            return "<{0.center.x}, {0.center.y}}, {0.radius}>".format(value)

        return None

    def get_prep_lookup(self, lookup_type, value):
        return NotImplementedError(self)

# Try to load South's model inspector
try:
    from south.modelsinspector import add_introspection_rules
    add_introspection_rules([], ['^postgres_geometry\.fields\.\w+Field'])

except ImportError:
    pass
<|endoftext|>