<commit_before>#!/usr/bin/python

import sys
import re
import json

class NginxServerMonitoring():
    def __init__(self, config_data) :
        self.data = {}
        self._config_data_ = config_data
        self.data['plugin_version'] = self._config_data_['plugin_version']
        self.data['heartbeat_required'] = self._config_data_['heartbeat_required']

    def _get_request_data_(self):
        PYTHON_MAJOR_VERSION = sys.version_info[0]
        if PYTHON_MAJOR_VERSION == 3:
            import urllib.request as urlconnection
        elif PYTHON_MAJOR_VERSION == 2:
            import urllib2 as urlconnection

        try:
            url = self._config_data_['url']

            if self._config_data_['username'] and self._config_data_['password']:
                password_mgr = urlconnection.HTTPPasswordMgrWithDefaultRealm()
                password_mgr.add_password(None, url, self._config_data_['username'], self._config_data_['password'])
                auth_handler = urlconnection.HTTPBasicAuthHandler(password_mgr)
                proxy_support = urlconnection.ProxyHandler({})
                opener = urlconnection.build_opener(auth_handler, proxy_support)
                urlconnection.install_opener(opener)

            response = urlconnection.urlopen(url, timeout=self._config_data_['timeout'])
            return response.read()
        except Exception as e:
            self.data['status'] = 0
            #self.data['msg'] = str(e.code) + " " + str(e.reason)
            self.data['msg'] = str(e)

    def _collect_metrics_(self):
        output = self._get_request_data_()
        if output == None : return self.data

        output = output.decode('utf-8')
        active_con = re.search(r'Active connections:\s+(\d+)', output)
        read_writes = re.search(r'Reading: (\d+)\s+Writing: (\d+)\s+Waiting: (\d+)', output)
        per_s_connections = re.search(r'\s*(\d+)\s+(\d+)\s+(\d+)', output)

        if active_con: self. data['active_connection'] = int(active_con.group(1)) # current active client connections including Waiting connections.
        if read_writes:
            reading, writing, waiting = read_writes.groups()
            self.data['reading']=reading # The current number of connections where nginx is reading the request header.
            self.data['writing']=writing # The current number of connections where nginx is writing the response back to the client.
            self.data['waiting']=waiting # The current number of idle client connections waiting for a request.

        if per_s_connections:
            conn = int(per_s_connections.group(1)) # The total number of accepted client connections.
            handled = int(per_s_connections.group(2)) # The total number of handled connections. Generally, same as accepts unless some resource limits have been reached (for example, the worker_connections limit).
            requests = int(per_s_connections.group(3)) # The total number of client requests.

            self.data['total_request'] = requests
            self.data['connection_opened']= handled
            self.data['connection_dropped'] = (conn - handled)

        return self.data

def _load_args_():
    _config_data_ = {}
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--nginx_status_url', help="nginx_status_url",type=str,default="http://localhost/nginx_status")
    parser.add_argument('--username', help='username', type=str, default=None)
    parser.add_argument('--password', help='password', type=str, default=None)

    parser.add_argument('--timeout', help ="timeout",type=int,default=60)

    parser.add_argument('--plugin_version', help='plugin_version', type=int,  nargs='?', default=1)
    parser.add_argument('--heartbeat', help='is heartbeat enabled', type=bool, nargs='?', default=True)

    args = parser.parse_args()

    _config_data_['url'] = args.nginx_status_url
    _config_data_['username'] = args.username
    _config_data_['password'] = args.password
    _config_data_['timeout'] = args.timeout
    _config_data_['plugin_version'] = args.plugin_version
    _config_data_['heartbeat_required'] = args.heartbeat

    return _config_data_

if __name__ == '__main__':
    _config_data_ =  _load_args_()
    nginx = NginxServerMonitoring(_config_data_)
    result = nginx._collect_metrics_()
    print(json.dumps(result, indent=4, sort_keys=True))

<commit_msg>Update nginx.py<commit_after>#!/usr/bin/python

import sys
import re
import json

class NginxServerMonitoring():
    def __init__(self, config_data) :
        self.data = {}
        self._config_data_ = config_data
        self.data['plugin_version'] = self._config_data_['plugin_version']
        self.data['heartbeat_required'] = self._config_data_['heartbeat_required']
        self.data['applog']=self._config_data_['applog']

    def _get_request_data_(self):
        PYTHON_MAJOR_VERSION = sys.version_info[0]
        if PYTHON_MAJOR_VERSION == 3:
            import urllib.request as urlconnection
        elif PYTHON_MAJOR_VERSION == 2:
            import urllib2 as urlconnection

        try:
            url = self._config_data_['url']

            if self._config_data_['username'] and self._config_data_['password']:
                password_mgr = urlconnection.HTTPPasswordMgrWithDefaultRealm()
                password_mgr.add_password(None, url, self._config_data_['username'], self._config_data_['password'])
                auth_handler = urlconnection.HTTPBasicAuthHandler(password_mgr)
                proxy_support = urlconnection.ProxyHandler({})
                opener = urlconnection.build_opener(auth_handler, proxy_support)
                urlconnection.install_opener(opener)

            response = urlconnection.urlopen(url, timeout=self._config_data_['timeout'])
            return response.read()
        except Exception as e:
            self.data['status'] = 0
            #self.data['msg'] = str(e.code) + " " + str(e.reason)
            self.data['msg'] = str(e)

    def _collect_metrics_(self):
        output = self._get_request_data_()
        if output == None : return self.data

        output = output.decode('utf-8')
        active_con = re.search(r'Active connections:\s+(\d+)', output)
        read_writes = re.search(r'Reading: (\d+)\s+Writing: (\d+)\s+Waiting: (\d+)', output)
        per_s_connections = re.search(r'\s*(\d+)\s+(\d+)\s+(\d+)', output)

        if active_con: self. data['Currently active client connections'] = int(active_con.group(1)) # current active client connections including Waiting connections.
        if read_writes:
            reading, writing, waiting = read_writes.groups()
            self.data['Number of connections where nginx is reading the request header']=reading # The current number of connections where nginx is reading the request header.
            self.data['Number of connections where nginx is writing the response back to the client']=writing # The current number of connections where nginx is writing the response back to the client.
            self.data['Number of idle client connections waiting for a request']=waiting # The current number of idle client connections waiting for a request.

        if per_s_connections:
            conn = int(per_s_connections.group(1)) # The total number of accepted client connections.
            handled = int(per_s_connections.group(2)) # The total number of handled connections. Generally, same as accepts unless some resource limits have been reached (for example, the worker_connections limit).
            requests = int(per_s_connections.group(3)) # The total number of client requests.

            self.data['Count of client requests'] = requests
            self.data['Count of successful client connections']= handled
            self.data['Count of dropped connections '] = (conn - handled)

        return self.data

def _load_args_():
    _config_data_ = {}
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--nginx_status_url', help="nginx_status_url",type=str,default="http://localhost/nginx_status")
    parser.add_argument('--username', help='username', type=str, default=None)
    parser.add_argument('--password', help='password', type=str, default=None)

    parser.add_argument('--timeout', help ="timeout",type=int,default=60)

    parser.add_argument('--plugin_version', help='plugin_version', type=int,  nargs='?', default=1)
    parser.add_argument('--heartbeat', help='is heartbeat enabled', type=bool, nargs='?', default=True)
    parser.add_argument('--logs_enabled', help='enable log collection for this plugin application',default="False")
    parser.add_argument('--log_type_name', help='Display name of the log type', nargs='?', default=None)
    parser.add_argument('--log_file_path', help='list of comma separated log file paths', nargs='?', default=None)

    args = parser.parse_args()

    _config_data_['url'] = args.nginx_status_url
    _config_data_['username'] = args.username
    _config_data_['password'] = args.password
    _config_data_['timeout'] = args.timeout
    _config_data_['plugin_version'] = args.plugin_version
    _config_data_['heartbeat_required'] = args.heartbeat

    logsenabled=args.logs_enabled
    logtypename=args.log_type_name
    logfilepath=args.log_file_path

    applog={}
    if(logsenabled in ['True', 'true', '1']):
        applog["logs_enabled"]=True
        applog["log_type_name"]=logtypename
        applog["log_file_path"]=logfilepath
    else:
        applog["logs_enabled"]=False
    _config_data_['applog'] = applog

    return _config_data_

if __name__ == '__main__':
    _config_data_ =  _load_args_()
    nginx = NginxServerMonitoring(_config_data_)
    result = nginx._collect_metrics_()
    print(json.dumps(result, indent=4, sort_keys=True))

<|endoftext|>


<commit_before>import requests
import re

from bs4 import BeautifulSoup

useUserAgent = "Mozilla/5.0 (Windows NT 10.0; rv:50.0) Gecko/20100101 Firefox/50.0"

class CurseAPI:
    def __init__(self):
        self.baseUrl = "https://mods.curse.com/mc-mods/minecraft"
        self.forgeUrl = "https://minecraft.curseforge.com/projects"
        self.session = requests.Session()
        self.session.headers.update({"User-Agent": useUserAgent})

    def getModList(self, version="", page=0):
        parsed = self.get(params = {
            "filter-project-game-version": version,
            "page": page
        })
        projects = parsed.body.find_all("ul", attrs = { "class": "project-listing" })[1]
        projects = [CurseProject(i) for i in projects.select("li > ul")]
        return projects

    def getFiles(self, pid):
        parsed = self.get(path="/{}/files".format(pid), host=self.forgeUrl)
        return [CurseFile(i) for i in parsed.select(".project-file-list-item")]

    def getVersionList(self):
        parsed = self.get()
        options = parsed.select("#filter-project-game-version > option")
        return [i["value"] for i in options if i["value"] != ""]

    def get(self, params={}, path="", host=False):
        if not host:
            host = self.baseUrl
        html = self.session.get(host+path, params=params).text
        return BeautifulSoup(html, "html.parser")

class CurseProject:
    def __init__(self, element):
        self.el = element

        self.title = self.getContent("h4 > a")
        self.id = self.el.select("h4 > a")[0]["href"].split("/")[-1].split("-")[0]

        self.updated = self.getContent(".updated")[8:]
        self.created = self.getContent(".updated", 1)[8:]

        self.monthly = int(self.getContent(".average-downloads")[:-8].replace(',', ''))
        self.total = int(self.getContent(".download-total")[:-6].replace(',', ''))

        self.likes = int(self.getContent(".grats")[:-6].replace(',', ''))

        self.latestVersion = self.getContent(".version")[10:]

    def getContent(self, selector, index=0):
        return self.el.select(selector)[index].contents[0]

class CurseFile:
    def __init__(self, element):
        self.el = element

        self.name = self.getContent(".project-file-name-container > a")

        self.releaseType = self.getThing(".project-file-release-type > div", "title")
        self.uploaded = self.getContent(".standard-datetime")

        self.url = self.getThing(".fa-icon-download", "href")
        self.size = float(self.getContent(".project-file-size")[14:-13])

        self.version = self.getContent(".version-label")

        self.downloads = int(self.getContent(".project-file-downloads")[14:-10].replace(',', ''))

    def getThing(self, selector, tag, index=0):
        return self.el.select(selector)[index][tag]

    def getContent(self, selector, index=0):
        return self.el.select(selector)[index].contents[0]

curse = CurseAPI()
<commit_msg>mod API complete<commit_after>import requests
import re

from bs4 import BeautifulSoup

useUserAgent = "Mozilla/5.0 (Windows NT 10.0; rv:50.0) Gecko/20100101 Firefox/50.0"

class CurseAPI:
    def __init__(self):
        self.baseUrl = "https://mods.curse.com/mc-mods/minecraft"
        self.forgeUrl = "https://minecraft.curseforge.com"
        self.session = requests.Session()
        self.session.headers.update({"User-Agent": useUserAgent})

    def getModList(self, version="", page=0):
        parsed = self.get(params = {
            "filter-project-game-version": version,
            "page": page
        })
        projects = parsed.body.find_all("ul", attrs = { "class": "project-listing" })[1]
        projects = [CurseProject(i) for i in projects.select("li > ul")]
        return projects

    def getFiles(self, pid):
        parsed = self.get(path="/projects/{}/files".format(pid), host=self.forgeUrl)
        return [CurseFile(i) for i in parsed.select(".project-file-list-item")]

    def getVersionList(self):
        parsed = self.get()
        options = parsed.select("#filter-project-game-version > option")
        return [i["value"] for i in options if i["value"] != ""]

    def get(self, params={}, path="", host=False):
        if not host:
            host = self.baseUrl
        html = self.session.get(host+path, params=params).text
        return BeautifulSoup(html, "html.parser")

    def downloadFile(self, url, filepath):
        r = self.session.get(url, stream=True)
        with open(filepath, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:
                    f.write(chunk)

class CurseProject:
    def __init__(self, element):
        self.el = element

        self.title = self.getContent("h4 > a")
        self.id = self.el.select("h4 > a")[0]["href"].split("/")[-1].split("-")[0]

        self.updated = self.getContent(".updated")[8:]
        self.created = self.getContent(".updated", 1)[8:]

        self.monthly = int(self.getContent(".average-downloads")[:-8].replace(',', ''))
        self.total = int(self.getContent(".download-total")[:-6].replace(',', ''))

        self.likes = int(self.getContent(".grats")[:-6].replace(',', ''))

        self.latestVersion = self.getContent(".version")[10:]

    def getContent(self, selector, index=0):
        return self.el.select(selector)[index].contents[0]

class CurseFile:
    def __init__(self, element):
        self.el = element

        self.name = self.getContent(".project-file-name-container > a")

        self.releaseType = self.getThing(".project-file-release-type > div", "title")
        self.uploaded = self.getContent(".standard-datetime")

        self.url = self.getThing(".fa-icon-download", "href")
        self.size = float(self.getContent(".project-file-size")[14:-13])

        self.version = self.getContent(".version-label")

        self.downloads = int(self.getContent(".project-file-downloads")[14:-10].replace(',', ''))

    def getThing(self, selector, tag, index=0):
        return self.el.select(selector)[index][tag]

    def getContent(self, selector, index=0):
        return self.el.select(selector)[index].contents[0]

curse = CurseAPI()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2012, 2013 Robert Martens <robert.martens@gmail.com>
# See file COPYING.txt for MIT license terms.

"""
Find floating point vertex coordinates in a Source engine .vmf file.

Looks through a .vmf for any brush planes whose vertices have
non-integer coordinates, printing a list to stdout and optionally
writing out a new file with rounded values. As a side effect of its
brute force regex search, vertalert only checks enabled VisGroups.

"""

import decimal
import os
import re
import sys

def get_dev(coord):
    orig = decimal.Decimal(coord)
    rounded = orig.quantize(1, decimal.ROUND_HALF_EVEN)
    return max(rounded, orig) - min(rounded, orig)

def get_max_dev(planes):
    """
    Search a brush's planes for the largest deviation any of its
    vertices' coordinates make from the nearest integer.

    Keyword arguments:
        planes: list of strings representing the planes to check

    """
    floats = []
    for plane in planes:
        # We're only looking for coordinates that were written to the
        # VMF as floating point values, perhaps in scientific notation.
        floats += re.findall(r'-?\d+\.\d+e?-?\d*', plane)
    devs = []
    for coord in floats:
        devs.append(get_dev(coord))
    return max(devs)

def fix_plane(plane, regex):
    """
    Use 'regex' pattern to find floating point coordinates in 'plane',
    round to nearest integer, and return corrected plane string.

    Keyword arguments:
        plane: string to search for floats
        regex: Regular Expression pattern to use for search

    """
    floats = re.findall(regex, plane)
    plane_new = plane
    for coord in floats:
        orig = decimal.Decimal(coord)
        rounded = orig.quantize(1, decimal.ROUND_HALF_EVEN)
        # I replace str(coord) instead of orig here, since
        # that would miss values using scientific notation.
        plane_new = plane_new.replace(str(coord), str(rounded), 1)
    return plane_new

def fix_brushes(brushes, thresh, vmf_in):
    """
    Find and fix brushes with floating point plane vertex coordinates.

    Returns a tuple containing the total number of brushes with floats,
    a list of the greatest deviation any of a brush's coordinates makes
    from the nearest integer, and a fixed version of vmf_in. Corrects
    only deviations less than 'thresh'.

    Keyword arguments:
        brushes: list of brush strings to search
        thresh: threshold below which to ignore/round coordinates
        vmf_in: string containing input VMF contents

    """
    vmf_out = vmf_in
    float_brushes = 0
    percent = len(brushes) / 100.0
    devs = []
    for i, brush in enumerate(brushes):
        brush_id = int(re.search(r'"id"\s"(\d+)"', brush).group(1))
        float_planes = []
        for plane in re.findall(r'"plane"\s".*?"', brush, re.DOTALL):
            if '.' in plane:
                float_planes.append(plane)
        if not float_planes:
            continue

        max_dev = get_max_dev(float_planes)
        if max_dev < thresh:
            brush_new = brush
            for plane in float_planes:
                plane_new = fix_plane(plane, r'-?\d+\.\d+e?-?\d*')
                brush_new = brush_new.replace(plane, plane_new)
            vmf_out = vmf_out.replace(brush, brush_new)
            float_brushes += 1
        else:
            devs.append((brush_id, max_dev))
        sys.stdout.write('\r%s%% complete' % str(int(i / percent)))
        sys.stdout.flush()
    sys.stdout.write("\r             \n")
    sys.stdout.flush()
    return (float_brushes, devs, vmf_out)

def print_dev_table(suspects, rounded_count, fix):
    """
    Print, to stdout, a table displaying each brush ID and its
    coordinates' maximum deviation from the nearest integer.

    Keyword arguments:
        sorted_devs: a list of tuples pairing brush id with max deviation

    """
    if suspects:
        suspects = sorted(suspects, key=lambda suspect: suspect[-1])
        max_id_width = len(str(max(suspects[0])))
        left_w = max(max_id_width, len("Suspect id"))
        header = ("Suspect ID").rjust(left_w) + '  ' + "Max dev" + '\n'
        sys.stdout.write(header)
        sys.stdout.write('-' * (len(header) - 1) + '\n')
        for suspect in suspects:
            suspect_str = str(suspect[0]).rjust(left_w) + "  " + str(suspect[1]) + "\n"
            sys.stdout.write(suspect_str)
            sys.stdout.flush()
        sys.stdout.write('\n')

    if len(suspects) == 1:
        warn_suffix = ""
    else:
        warn_suffix = "es"
    if rounded_count == 1:
        action_suffix = ""
    else:
        action_suffix = "es"
    if fix:
        action = " automatically rounded"
    else:
        action = " ignored"

    sys.stdout.write(str(rounded_count) + " brush" +
                     action_suffix + action + '\n')
    sys.stdout.write(str(len(suspects)) + " suspect brush" +
                     warn_suffix + " remaining\n")
    sys.stdout.flush()

def vertalert(file_in, fix=False, fixname=None, thresh=None):
    """
    Find, and optionally fix, floating point plane coordinates in a
    Source engine .vmf file.

    Prints a list to stdout of all values greater than or equal to
    thresh, rounds (when using --fix) or ignores all other values.

    Keyword arguments:
        file_in: .vmf file to check
        fix: write out a new file with rounded coordinates (default False)
        fixname: filename when using --fix (default appends _VERTALERT)
        thresh: threshold below which to ignore/round coordinates (default 0.2)

    Please note this function currently only checks enabled VisGroups.

    """
    if not os.path.exists(file_in):
        sys.stderr.write("Could not find " + file_in + "!\n")
        return -1
    if os.path.splitext(file_in)[1] != ".vmf":
        sys.stderr.write("Input must be a .vmf file!\n")
        return -1
    if fixname is None:
        in_name_split = os.path.splitext(file_in)
        fixname = in_name_split[0] + "_VERTALERT" + in_name_split[1]
    if thresh is None:
        thresh = decimal.Context().create_decimal('0.2')

    with open(file_in, 'r') as vmf:
        vmf_in = vmf.read()

    # I no longer read the input in universal line ending mode above,
    # as that changes the type of line endings written to the output
    # when using --fix. Although Hammer can deal with at least Windows
    # and Unix endings, I prefer to write out the same thing I read in.
    # The \r?\n in this regular expression became necessary to ensure
    # Windows, Linux and OS X worked the same way. The pattern is that
    # of the beginning of a solid entry, whose VisGroup is enabled, and
    # breaks down as follows:
    #
    # solid - The word 'solid'.
    # \r?\n - Zero or one carriage return followed by one newline. In
    #         Windows, \n is what \r\n is under Linux and OS X.
    # \t{ - Tab, open curly brace
    # .*? - Zero or more of any character (including newlines, thanks
    #       to my use of re.DOTALL in the call to findall), non-greedy.
    # \r?\n\t} - Zero or one carriage return, one newline, one tab,
    #            closing curly brace.
    brushes = re.findall(r'solid\r?\n\t{.*?\r?\n\t}', vmf_in, re.DOTALL)
    float_brushes, devs, vmf_out = fix_brushes(brushes, thresh, vmf_in)

    print_dev_table(devs, float_brushes, fix)

    if fix:
        with open(fixname, 'w') as vmf:
            vmf.write(vmf_out)
        sys.stdout.write("\nWrote " + fixname + "!\n")
        sys.stdout.flush()

    return 0

if __name__ == '__main__':
    import argparse
    PARSER = argparse.ArgumentParser(description="VertAlert 0.2.1")
    PARSER.add_argument("input", help="VMF to check")
    PARSER.add_argument(
        "-f", "--fix", help="write fixed up VMF", action="store_true")
    PARSER.add_argument(
        "-fn", "--fixname",
        help="filename to use with --fix (default appends _VERTALERT)")
    PARSER.add_argument(
        "-t", "--thresh",
        type=decimal.Decimal,
        help="threshold below which to ignore/round coordinates (default 0.2)")
    ARGS = PARSER.parse_args()

    vertalert(ARGS.input, ARGS.fix, ARGS.fixname, ARGS.thresh)
<commit_msg>Switch get_dev to divide/quant/multiply approach<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2012, 2013 Robert Martens <robert.martens@gmail.com>
# See file COPYING.txt for MIT license terms.

"""
Find floating point vertex coordinates in a Source engine .vmf file.

Looks through a .vmf for any brush planes whose vertices have
non-integer coordinates, printing a list to stdout and optionally
writing out a new file with rounded values. As a side effect of its
brute force regex search, vertalert only checks enabled VisGroups.

"""

import decimal
import os
import re
import sys

def get_dev(coord, snap):
    orig = decimal.Decimal(coord)
    rounded = (orig / snap).quantize(1, decimal.ROUND_HALF_EVEN) * snap
    return max(rounded, orig) - min(rounded, orig)

def get_max_dev(planes):
    """
    Search a brush's planes for the largest deviation any of its
    vertices' coordinates make from the nearest integer.

    Keyword arguments:
        planes: list of strings representing the planes to check

    """
    floats = []
    for plane in planes:
        # We're only looking for coordinates that were written to the
        # VMF as floating point values, perhaps in scientific notation.
        floats += re.findall(r'-?\d+\.\d+e?-?\d*', plane)
    devs = []
    for coord in floats:
        devs.append(get_dev(coord, 1))
    return max(devs)

def fix_plane(plane, regex):
    """
    Use 'regex' pattern to find floating point coordinates in 'plane',
    round to nearest integer, and return corrected plane string.

    Keyword arguments:
        plane: string to search for floats
        regex: Regular Expression pattern to use for search

    """
    floats = re.findall(regex, plane)
    plane_new = plane
    for coord in floats:
        orig = decimal.Decimal(coord)
        rounded = orig.quantize(1, decimal.ROUND_HALF_EVEN)
        # I replace str(coord) instead of orig here, since
        # that would miss values using scientific notation.
        plane_new = plane_new.replace(str(coord), str(rounded), 1)
    return plane_new

def fix_brushes(brushes, thresh, vmf_in):
    """
    Find and fix brushes with floating point plane vertex coordinates.

    Returns a tuple containing the total number of brushes with floats,
    a list of the greatest deviation any of a brush's coordinates makes
    from the nearest integer, and a fixed version of vmf_in. Corrects
    only deviations less than 'thresh'.

    Keyword arguments:
        brushes: list of brush strings to search
        thresh: threshold below which to ignore/round coordinates
        vmf_in: string containing input VMF contents

    """
    vmf_out = vmf_in
    float_brushes = 0
    percent = len(brushes) / 100.0
    devs = []
    for i, brush in enumerate(brushes):
        brush_id = int(re.search(r'"id"\s"(\d+)"', brush).group(1))
        float_planes = []
        for plane in re.findall(r'"plane"\s".*?"', brush, re.DOTALL):
            if '.' in plane:
                float_planes.append(plane)
        if not float_planes:
            continue

        max_dev = get_max_dev(float_planes)
        if max_dev < thresh:
            brush_new = brush
            for plane in float_planes:
                plane_new = fix_plane(plane, r'-?\d+\.\d+e?-?\d*')
                brush_new = brush_new.replace(plane, plane_new)
            vmf_out = vmf_out.replace(brush, brush_new)
            float_brushes += 1
        else:
            devs.append((brush_id, max_dev))
        sys.stdout.write('\r%s%% complete' % str(int(i / percent)))
        sys.stdout.flush()
    sys.stdout.write("\r             \n")
    sys.stdout.flush()
    return (float_brushes, devs, vmf_out)

def print_dev_table(suspects, rounded_count, fix):
    """
    Print, to stdout, a table displaying each brush ID and its
    coordinates' maximum deviation from the nearest integer.

    Keyword arguments:
        sorted_devs: a list of tuples pairing brush id with max deviation

    """
    if suspects:
        suspects = sorted(suspects, key=lambda suspect: suspect[-1])
        max_id_width = len(str(max(suspects[0])))
        left_w = max(max_id_width, len("Suspect id"))
        header = ("Suspect ID").rjust(left_w) + '  ' + "Max dev" + '\n'
        sys.stdout.write(header)
        sys.stdout.write('-' * (len(header) - 1) + '\n')
        for suspect in suspects:
            suspect_str = str(suspect[0]).rjust(left_w) + "  " + str(suspect[1]) + "\n"
            sys.stdout.write(suspect_str)
            sys.stdout.flush()
        sys.stdout.write('\n')

    if len(suspects) == 1:
        warn_suffix = ""
    else:
        warn_suffix = "es"
    if rounded_count == 1:
        action_suffix = ""
    else:
        action_suffix = "es"
    if fix:
        action = " automatically rounded"
    else:
        action = " ignored"

    sys.stdout.write(str(rounded_count) + " brush" +
                     action_suffix + action + '\n')
    sys.stdout.write(str(len(suspects)) + " suspect brush" +
                     warn_suffix + " remaining\n")
    sys.stdout.flush()

def vertalert(file_in, fix=False, fixname=None, thresh=None):
    """
    Find, and optionally fix, floating point plane coordinates in a
    Source engine .vmf file.

    Prints a list to stdout of all values greater than or equal to
    thresh, rounds (when using --fix) or ignores all other values.

    Keyword arguments:
        file_in: .vmf file to check
        fix: write out a new file with rounded coordinates (default False)
        fixname: filename when using --fix (default appends _VERTALERT)
        thresh: threshold below which to ignore/round coordinates (default 0.2)

    Please note this function currently only checks enabled VisGroups.

    """
    if not os.path.exists(file_in):
        sys.stderr.write("Could not find " + file_in + "!\n")
        return -1
    if os.path.splitext(file_in)[1] != ".vmf":
        sys.stderr.write("Input must be a .vmf file!\n")
        return -1
    if fixname is None:
        in_name_split = os.path.splitext(file_in)
        fixname = in_name_split[0] + "_VERTALERT" + in_name_split[1]
    if thresh is None:
        thresh = decimal.Context().create_decimal('0.2')

    with open(file_in, 'r') as vmf:
        vmf_in = vmf.read()

    # I no longer read the input in universal line ending mode above,
    # as that changes the type of line endings written to the output
    # when using --fix. Although Hammer can deal with at least Windows
    # and Unix endings, I prefer to write out the same thing I read in.
    # The \r?\n in this regular expression became necessary to ensure
    # Windows, Linux and OS X worked the same way. The pattern is that
    # of the beginning of a solid entry, whose VisGroup is enabled, and
    # breaks down as follows:
    #
    # solid - The word 'solid'.
    # \r?\n - Zero or one carriage return followed by one newline. In
    #         Windows, \n is what \r\n is under Linux and OS X.
    # \t{ - Tab, open curly brace
    # .*? - Zero or more of any character (including newlines, thanks
    #       to my use of re.DOTALL in the call to findall), non-greedy.
    # \r?\n\t} - Zero or one carriage return, one newline, one tab,
    #            closing curly brace.
    brushes = re.findall(r'solid\r?\n\t{.*?\r?\n\t}', vmf_in, re.DOTALL)
    float_brushes, devs, vmf_out = fix_brushes(brushes, thresh, vmf_in)

    print_dev_table(devs, float_brushes, fix)

    if fix:
        with open(fixname, 'w') as vmf:
            vmf.write(vmf_out)
        sys.stdout.write("\nWrote " + fixname + "!\n")
        sys.stdout.flush()

    return 0

if __name__ == '__main__':
    import argparse
    PARSER = argparse.ArgumentParser(description="VertAlert 0.2.1")
    PARSER.add_argument("input", help="VMF to check")
    PARSER.add_argument(
        "-f", "--fix", help="write fixed up VMF", action="store_true")
    PARSER.add_argument(
        "-fn", "--fixname",
        help="filename to use with --fix (default appends _VERTALERT)")
    PARSER.add_argument(
        "-t", "--thresh",
        type=decimal.Decimal,
        help="threshold below which to ignore/round coordinates (default 0.2)")
    ARGS = PARSER.parse_args()

    vertalert(ARGS.input, ARGS.fix, ARGS.fixname, ARGS.thresh)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright © 2014—2016 Dontnod Entertainment

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
''' Unreal Engine 4 related stuff '''

import os
import platform
import logging

import nimp.build
import nimp.system

def load_config(env):
    ''' Loads Unreal specific configuration values on env before parsing
        command-line arguments '''
    env.is_ue4 = hasattr(env, 'project_type') and env.project_type is 'UE4'
    return True

def load_arguments(env):
    ''' Loads Unreal specific environment parameters. '''
    if hasattr(env, 'project_type') and env.project_type == 'UE4':
        return _ue4_load_arguments(env)

    return True

def build(env):
    ''' Builds an Unreal Engine Project. config and platform arguments should
        be set on environment in order to call this function. You can use
        `nimp.environment.add_arguments` and `nimp.build.add_arguments` to do
        so.'''
    if not _check_for_unreal(env):
        return False
    return _ue4_build(env)

def commandlet(env, command, *args, heartbeat = 0):
    ''' Runs an Unreal Engine commandlet. It can be usefull to run it through
        nimp as OutputDebugString will be redirected to standard output. '''
    if not _check_for_unreal(env):
        return False
    return _ue4_commandlet(env, command, *args, heartbeat = heartbeat)

def is_unreal4_available(env):
    ''' Returns a tuple containing unreal availability and a help text
        giving a reason of why it isn't if it's the case '''
    return env.is_ue4, ('No .nimp.conf configured for Unreal 4 was found in '
                        'this directory or one of its parents. Check that you '
                        'are launching nimp from inside an UE4 project directory'
                        ' and that you have a properly configured .nimp.conf. '
                        'Check documentation for more details on .nimp.conf.')
def _check_for_unreal(env):
    if not env.is_ue4:
        logging.error('This doesn\'t seems to be a supported project type.')
        logging.error('Check that you are launching nimp from an UE project')
        logging.error('directory, and that you have a .nimp.conf file in the')
        logging.error('project root directory (see documentation for further')
        logging.error('details)')
        return False
    return True

def _ue4_build(env):
    assert hasattr(env, 'ue4_config')
    assert env.ue4_config is not None

    if env.disable_unity:
        os.environ['UBT_bUseUnityBuild'] = 'false'

    if env.fastbuild:
        os.environ['UBT_bAllowFastBuild'] = 'true'
        os.environ['UBT_bUseUnityBuild'] = 'false'

    nimp.environment.execute_hook('prebuild', env)

    # Bootstrap if necessary
    if hasattr(env, 'bootstrap') and env.bootstrap:
        # Now generate project files
        if _ue4_generate_project(env) != 0:
            logging.error("Error generating UE4 project files")
            return False

    # Default to VS 2015, The Durango XDK does not support Visual Studio 2013 yet, so if UE4
    # detected it, it created VS 2012 project files and we have to use that
    # version to build the tools instead.
    vs_version = '14'
    try:
        for line in open(env.format(env.solution)):
            if ('# Visual Studio 2012' in line or
                '# Visual Studio 11' in line):
                vs_version = '11'
                break
            if ('# Visual Studio 2013' in line or
                '# Visual Studio 12' in line):
                vs_version = '12'
                break
            if ('# Visual Studio 2015' in line or
                '# Visual Studio 14' in line):
                vs_version = '14'
                break
    except IOError:
        pass

    # The main solution file
    solution = env.format(env.solution)

    # We’ll try to build all tools even in case of failure
    success = True

    # List of tools to build
    tools = [ 'UnrealHeaderTool' ]

    # Some tools are necessary even when not building tools...
    need_ps4devkitutil = False
    need_ps4mapfileutil = env.platform == 'ps4'
    need_xboxonepdbfileutil = env.platform == 'xboxone'

    if env.target == 'tools':

        tools += [ 'UnrealFrontend',
                   'UnrealFileServer',
                   'ShaderCompileWorker',
                   'UnrealPak',
                   'CrashReportClient' ]

        if env.platform != 'mac':
            tools += [ 'UnrealLightmass', ] # doesn’t build (yet?)

        if env.platform == 'linux':
            tools += [ 'CrossCompilerTool', ]

        if env.platform == 'win64':
            tools += [ 'DotNETUtilities',
                       'AutomationTool',
                       'SymbolDebugger' ]
            need_ps4devkitutil = True
            need_ps4mapfileutil = True
            need_xboxonepdbfileutil = True

    # Some tools are necessary even when not building tools...
    if need_ps4devkitutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/PS4/PS4DevKitUtil/PS4DevKitUtil.csproj'))):
        tools += [ 'PS4DevKitUtil' ]

    if need_ps4mapfileutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/PS4/PS4MapFileUtil/PS4MapFileUtil.Build.cs'))):
        tools += [ 'PS4MapFileUtil' ]

    if need_xboxonepdbfileutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/XboxOne/XboxOnePDBFileUtil/XboxOnePDBFileUtil.Build.cs'))):
        tools += [ 'XboxOnePDBFileUtil' ]

    # Build tools from the main solution
    for tool in tools:
        if not _ue4_build_project(env, solution, tool,
                                  'Mac' if env.platform == 'mac'
                                  else 'Linux' if env.platform == 'linux'
                                  else 'Win64',
                                  'Shipping' if tool == 'CrashReportClient'
                                  else 'Development',
                                  vs_version, 'Build'):
            logging.error("Could not build %s", tool)
            success = False

    # Build tools from other solutions or with other flags
    if env.target == 'tools':

        if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Programs/NetworkProfiler/NetworkProfiler.sln'),
                                  'Any CPU', 'Development',
                                  vs_version=vs_version,
                                  target='Build'):
            logging.error("Could not build NetworkProfiler")
            success = False

        if env.platform != 'win64':
            # On Windows this is part of the main .sln, but not on Linux…
            if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Programs/AutomationTool/AutomationTool_Mono.sln'),
                                      'Any CPU', 'Development',
                                      vs_version=vs_version,
                                      target='Build'):
                logging.error("Could not build AutomationTool")
                success = False

        if env.platform != 'mac':
            # This also builds AgentInterface.dll, needed by SwarmInterface.sln
            if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Programs/UnrealSwarm/UnrealSwarm.sln'),
                                      'Any CPU', 'Development',
                                      vs_version=vs_version,
                                      target='Build'):
                logging.error("Could not build UnrealSwarm")
                success = False

            if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Editor/SwarmInterface/DotNET/SwarmInterface.sln'),
                                      'Any CPU', 'Development',
                                      vs_version=vs_version,
                                      target='Build'):
                logging.error("Could not build SwarmInterface")
                success = False

        # These tools seem to be Windows only for now
        if env.platform == 'win64':

            if not _ue4_build_project(env, solution, 'BootstrapPackagedGame',
                                      'Win64', 'Shipping', vs_version, 'Build'):
                logging.error("Could not build BootstrapPackagedGame")
                success = False

            tmp = env.format('{root_dir}/Engine/Source/Programs/XboxOne/XboxOnePackageNameUtil/XboxOnePackageNameUtil.sln')
            if os.path.exists(nimp.system.sanitize_path(tmp)):
                if not nimp.build.vsbuild(tmp, 'x64', 'Development',
                                          vs_version='11',
                                          target='Build') \
                   and not nimp.build.vsbuild(tmp, 'x64', 'Development',
                                              vs_version='14',
                                              target='Build'):
                    logging.error("Could not build XboxOnePackageNameUtil")
                    success = False

    if not success:
        return success

    if env.target == 'game':
        if not _ue4_build_project(env, solution, env.game, env.ue4_platform,
                                  env.ue4_config, vs_version, 'Build'):
            logging.error("Could not build game project")
            success = False

    if env.target == 'editor':
        if env.platform in ['linux', 'mac']:
            project = env.game + 'Editor'
            config = env.ue4_config
        else:
            project = env.game
            config = env.ue4_config + ' Editor'

        if not _ue4_build_project(env, solution, project, env.ue4_platform,
                                  config, vs_version, 'Build'):
            logging.error("Could not build editor project")
            success = False

    if success:
        nimp.environment.execute_hook('postbuild', env)

    return success

def _ue4_commandlet(env, command, *args, heartbeat = 0):
    ''' Runs an UE4 commandlet '''
    if nimp.system.is_windows():
        exe = 'Engine/Binaries/Win64/UE4Editor.exe'
    elif platform.system() == 'Darwin':
        exe = 'Engine/Binaries/Mac/UE4Editor'
    else:
        exe = 'Engine/Binaries/Linux/UE4Editor'

    cmdline = [nimp.system.sanitize_path(os.path.join(env.format(env.root_dir), exe)),
               env.game,
               '-run=%s' % command]

    cmdline += list(args)
    cmdline += ['-nopause', '-buildmachine', '-forcelogflush', '-unattended', '-noscriptcheck']

    return nimp.system.call_process('.', cmdline, heartbeat = heartbeat) == 0

def _ue4_generate_project(env):
    if nimp.system.is_windows():
        return nimp.system.call_process(env.root_dir, ['cmd', '/c', 'GenerateProjectFiles.bat', '-2015'])
    else:
        return nimp.system.call_process(env.root_dir, ['/bin/sh', './GenerateProjectFiles.sh'])

def _ue4_build_project(env, sln_file, project, build_platform,
                       configuration, vs_version, target = 'Rebuild'):

    if nimp.system.is_windows():
        return nimp.build.vsbuild(sln_file, build_platform, configuration,
                                  project=project,
                                  vs_version=vs_version,
                                  target=target)

    if platform.system() == 'Darwin':
        host_platform = 'Mac'
    else:
        host_platform = 'Linux'

    return nimp.system.call_process(env.root_dir,
                                    ['/bin/sh', './Engine/Build/BatchFiles/%s/Build.sh' % (host_platform),
                                     project, build_platform, configuration]) == 0

def _ue4_load_arguments(env):
    ''' Sets some variables for use with unreal 4 '''
    def _get_ue4_config(config):
        configs = { "debug"    : "Debug",
                    "devel"    : "Development",
                    "test"     : "Test",
                    "shipping" : "Shipping", }
        if config not in configs:
            logging.warning('Unsupported UE4 build config “%s”', config)
            return None
        return configs[config]

    def _get_ue4_platform(in_platform):
        platforms = { "ps4"     : "PS4",
                      "xboxone" : "XboxOne",
                      "win64"   : "Win64",
                      "win32"   : "Win32",
                      "linux"   : "Linux",
                      "mac"     : "Mac", }
        if in_platform not in platforms:
            logging.warning('Unsupported UE4 build platform “%s”', in_platform)
            return None
        return platforms[in_platform]

    if hasattr(env, 'platform'):
        env.ue4_platform = '+'.join(map(_get_ue4_platform, env.platform.split('+')))

    if hasattr(env, 'configuration'):
        if env.configuration is None:
            env.configuration = 'devel'
        env.ue4_config = '+'.join(map(_get_ue4_config, env.configuration.split('+')))

    if not hasattr(env, 'target') or env.target is None and env.is_ue4:
        if env.platform in ['win64', 'mac', 'linux']:
            env.target = 'editor'
        else:
            env.target = 'game'

    return True
<commit_msg>Build AutoReporter as part of the tools.<commit_after># -*- coding: utf-8 -*-
# Copyright © 2014—2016 Dontnod Entertainment

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
''' Unreal Engine 4 related stuff '''

import os
import platform
import logging

import nimp.build
import nimp.system

def load_config(env):
    ''' Loads Unreal specific configuration values on env before parsing
        command-line arguments '''
    env.is_ue4 = hasattr(env, 'project_type') and env.project_type is 'UE4'
    return True

def load_arguments(env):
    ''' Loads Unreal specific environment parameters. '''
    if hasattr(env, 'project_type') and env.project_type == 'UE4':
        return _ue4_load_arguments(env)

    return True

def build(env):
    ''' Builds an Unreal Engine Project. config and platform arguments should
        be set on environment in order to call this function. You can use
        `nimp.environment.add_arguments` and `nimp.build.add_arguments` to do
        so.'''
    if not _check_for_unreal(env):
        return False
    return _ue4_build(env)

def commandlet(env, command, *args, heartbeat = 0):
    ''' Runs an Unreal Engine commandlet. It can be usefull to run it through
        nimp as OutputDebugString will be redirected to standard output. '''
    if not _check_for_unreal(env):
        return False
    return _ue4_commandlet(env, command, *args, heartbeat = heartbeat)

def is_unreal4_available(env):
    ''' Returns a tuple containing unreal availability and a help text
        giving a reason of why it isn't if it's the case '''
    return env.is_ue4, ('No .nimp.conf configured for Unreal 4 was found in '
                        'this directory or one of its parents. Check that you '
                        'are launching nimp from inside an UE4 project directory'
                        ' and that you have a properly configured .nimp.conf. '
                        'Check documentation for more details on .nimp.conf.')
def _check_for_unreal(env):
    if not env.is_ue4:
        logging.error('This doesn\'t seems to be a supported project type.')
        logging.error('Check that you are launching nimp from an UE project')
        logging.error('directory, and that you have a .nimp.conf file in the')
        logging.error('project root directory (see documentation for further')
        logging.error('details)')
        return False
    return True

def _ue4_build(env):
    assert hasattr(env, 'ue4_config')
    assert env.ue4_config is not None

    if env.disable_unity:
        os.environ['UBT_bUseUnityBuild'] = 'false'

    if env.fastbuild:
        os.environ['UBT_bAllowFastBuild'] = 'true'
        os.environ['UBT_bUseUnityBuild'] = 'false'

    nimp.environment.execute_hook('prebuild', env)

    # Bootstrap if necessary
    if hasattr(env, 'bootstrap') and env.bootstrap:
        # Now generate project files
        if _ue4_generate_project(env) != 0:
            logging.error("Error generating UE4 project files")
            return False

    # Default to VS 2015, The Durango XDK does not support Visual Studio 2013 yet, so if UE4
    # detected it, it created VS 2012 project files and we have to use that
    # version to build the tools instead.
    vs_version = '14'
    try:
        for line in open(env.format(env.solution)):
            if ('# Visual Studio 2012' in line or
                '# Visual Studio 11' in line):
                vs_version = '11'
                break
            if ('# Visual Studio 2013' in line or
                '# Visual Studio 12' in line):
                vs_version = '12'
                break
            if ('# Visual Studio 2015' in line or
                '# Visual Studio 14' in line):
                vs_version = '14'
                break
    except IOError:
        pass

    # The main solution file
    solution = env.format(env.solution)

    # We’ll try to build all tools even in case of failure
    success = True

    # List of tools to build
    tools = [ 'UnrealHeaderTool' ]

    # Some tools are necessary even when not building tools...
    need_ps4devkitutil = False
    need_ps4mapfileutil = env.platform == 'ps4'
    need_xboxonepdbfileutil = env.platform == 'xboxone'

    if env.target == 'tools':

        tools += [ 'UnrealFrontend',
                   'UnrealFileServer',
                   'ShaderCompileWorker',
                   'UnrealPak',
                   'CrashReportClient' ]

        if env.platform != 'mac':
            tools += [ 'UnrealLightmass', ] # doesn’t build (yet?)

        if env.platform == 'linux':
            tools += [ 'CrossCompilerTool', ]

        if env.platform == 'win64':
            tools += [ 'DotNETUtilities',
                       'AutomationTool',
                       'SymbolDebugger' ]
            need_ps4devkitutil = True
            need_ps4mapfileutil = True
            need_xboxonepdbfileutil = True

    # Some tools are necessary even when not building tools...
    if need_ps4devkitutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/PS4/PS4DevKitUtil/PS4DevKitUtil.csproj'))):
        tools += [ 'PS4DevKitUtil' ]

    if need_ps4mapfileutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/PS4/PS4MapFileUtil/PS4MapFileUtil.Build.cs'))):
        tools += [ 'PS4MapFileUtil' ]

    if need_xboxonepdbfileutil and os.path.exists(nimp.system.sanitize_path(env.format('{root_dir}/Engine/Source/Programs/XboxOne/XboxOnePDBFileUtil/XboxOnePDBFileUtil.Build.cs'))):
        tools += [ 'XboxOnePDBFileUtil' ]

    # Build tools from the main solution
    for tool in tools:
        if not _ue4_build_project(env, solution, tool,
                                  'Mac' if env.platform == 'mac'
                                  else 'Linux' if env.platform == 'linux'
                                  else 'Win64',
                                  'Shipping' if tool == 'CrashReportClient'
                                  else 'Development',
                                  vs_version, 'Build'):
            logging.error("Could not build %s", tool)
            success = False

    # Build tools from other solutions or with other flags
    if env.target == 'tools':

        if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Programs/NetworkProfiler/NetworkProfiler.sln'),
                                  'Any CPU', 'Development',
                                  vs_version=vs_version,
                                  target='Build'):
            logging.error("Could not build NetworkProfiler")
            success = False

        if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Programs/CrashReport/CrashReport.sln'),
                                  'Any CPU', 'Release',
                                  project='AutoReporter',
                                  vs_version=vs_version,
                                  target='Build'):
            logging.error("Could not build AutoReporter")
            success = False

        if env.platform != 'win64':
            # On Windows this is part of the main .sln, but not on Linux…
            if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Programs/AutomationTool/AutomationTool_Mono.sln'),
                                      'Any CPU', 'Development',
                                      vs_version=vs_version,
                                      target='Build'):
                logging.error("Could not build AutomationTool")
                success = False

        if env.platform != 'mac':
            # This also builds AgentInterface.dll, needed by SwarmInterface.sln
            if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Programs/UnrealSwarm/UnrealSwarm.sln'),
                                      'Any CPU', 'Development',
                                      vs_version=vs_version,
                                      target='Build'):
                logging.error("Could not build UnrealSwarm")
                success = False

            if not nimp.build.vsbuild(env.format('{root_dir}/Engine/Source/Editor/SwarmInterface/DotNET/SwarmInterface.sln'),
                                      'Any CPU', 'Development',
                                      vs_version=vs_version,
                                      target='Build'):
                logging.error("Could not build SwarmInterface")
                success = False

        # These tools seem to be Windows only for now
        if env.platform == 'win64':

            if not _ue4_build_project(env, solution, 'BootstrapPackagedGame',
                                      'Win64', 'Shipping', vs_version, 'Build'):
                logging.error("Could not build BootstrapPackagedGame")
                success = False

            tmp = env.format('{root_dir}/Engine/Source/Programs/XboxOne/XboxOnePackageNameUtil/XboxOnePackageNameUtil.sln')
            if os.path.exists(nimp.system.sanitize_path(tmp)):
                if not nimp.build.vsbuild(tmp, 'x64', 'Development',
                                          vs_version='11',
                                          target='Build') \
                   and not nimp.build.vsbuild(tmp, 'x64', 'Development',
                                              vs_version='14',
                                              target='Build'):
                    logging.error("Could not build XboxOnePackageNameUtil")
                    success = False

    if not success:
        return success

    if env.target == 'game':
        if not _ue4_build_project(env, solution, env.game, env.ue4_platform,
                                  env.ue4_config, vs_version, 'Build'):
            logging.error("Could not build game project")
            success = False

    if env.target == 'editor':
        if env.platform in ['linux', 'mac']:
            project = env.game + 'Editor'
            config = env.ue4_config
        else:
            project = env.game
            config = env.ue4_config + ' Editor'

        if not _ue4_build_project(env, solution, project, env.ue4_platform,
                                  config, vs_version, 'Build'):
            logging.error("Could not build editor project")
            success = False

    if success:
        nimp.environment.execute_hook('postbuild', env)

    return success

def _ue4_commandlet(env, command, *args, heartbeat = 0):
    ''' Runs an UE4 commandlet '''
    if nimp.system.is_windows():
        exe = 'Engine/Binaries/Win64/UE4Editor.exe'
    elif platform.system() == 'Darwin':
        exe = 'Engine/Binaries/Mac/UE4Editor'
    else:
        exe = 'Engine/Binaries/Linux/UE4Editor'

    cmdline = [nimp.system.sanitize_path(os.path.join(env.format(env.root_dir), exe)),
               env.game,
               '-run=%s' % command]

    cmdline += list(args)
    cmdline += ['-nopause', '-buildmachine', '-forcelogflush', '-unattended', '-noscriptcheck']

    return nimp.system.call_process('.', cmdline, heartbeat = heartbeat) == 0

def _ue4_generate_project(env):
    if nimp.system.is_windows():
        return nimp.system.call_process(env.root_dir, ['cmd', '/c', 'GenerateProjectFiles.bat', '-2015'])
    else:
        return nimp.system.call_process(env.root_dir, ['/bin/sh', './GenerateProjectFiles.sh'])

def _ue4_build_project(env, sln_file, project, build_platform,
                       configuration, vs_version, target = 'Rebuild'):

    if nimp.system.is_windows():
        return nimp.build.vsbuild(sln_file, build_platform, configuration,
                                  project=project,
                                  vs_version=vs_version,
                                  target=target)

    if platform.system() == 'Darwin':
        host_platform = 'Mac'
    else:
        host_platform = 'Linux'

    return nimp.system.call_process(env.root_dir,
                                    ['/bin/sh', './Engine/Build/BatchFiles/%s/Build.sh' % (host_platform),
                                     project, build_platform, configuration]) == 0

def _ue4_load_arguments(env):
    ''' Sets some variables for use with unreal 4 '''
    def _get_ue4_config(config):
        configs = { "debug"    : "Debug",
                    "devel"    : "Development",
                    "test"     : "Test",
                    "shipping" : "Shipping", }
        if config not in configs:
            logging.warning('Unsupported UE4 build config “%s”', config)
            return None
        return configs[config]

    def _get_ue4_platform(in_platform):
        platforms = { "ps4"     : "PS4",
                      "xboxone" : "XboxOne",
                      "win64"   : "Win64",
                      "win32"   : "Win32",
                      "linux"   : "Linux",
                      "mac"     : "Mac", }
        if in_platform not in platforms:
            logging.warning('Unsupported UE4 build platform “%s”', in_platform)
            return None
        return platforms[in_platform]

    if hasattr(env, 'platform'):
        env.ue4_platform = '+'.join(map(_get_ue4_platform, env.platform.split('+')))

    if hasattr(env, 'configuration'):
        if env.configuration is None:
            env.configuration = 'devel'
        env.ue4_config = '+'.join(map(_get_ue4_config, env.configuration.split('+')))

    if not hasattr(env, 'target') or env.target is None and env.is_ue4:
        if env.platform in ['win64', 'mac', 'linux']:
            env.target = 'editor'
        else:
            env.target = 'game'

    return True
<|endoftext|>


<commit_before>import sublime
import sublime_plugin

import re

def find_in_range(view, term, start, end, flags=0):
    found = view.find(term, start, flags)
    if found and found.b <= end:
        return found

def find_all_in_range(view, term, start, end, flags=0):
    matches = []
    while True:
        m = find_in_range(view, term, start, end, flags)

        if m == sublime.Region(-1, -1):
            return matches

        if not m:
            return matches

        if m.end() > end or m.begin() < start:
            return matches

        matches.append(m)
        start = m.end()

def find_wrapping(view, term, start, end, flags=0, times=1):
    current_sel = view.sel()[0]
    # Search wrapping around the end of the buffer.
    for x in range(times):
        match = find_in_range(view, term, start, end, flags)
        # Start searching in the upper half of the buffer if we aren't doing it yet.
        if not match:
            start = 0
            end = current_sel.a
            match = find_in_range(view, term, start, end, flags)
            if not match:
                return
        # No luck in the whole buffer.
        elif not match:
            return
        start = match.b

    return match

def reverse_find_wrapping(view, term, start, end, flags=0, times=1):
    current_sel = view.sel()[0]
    # Search wrapping around the end of the buffer.
    for x in range(times):
        match = reverse_search(view, term, start, end, flags)
        # Start searching in the lower half of the buffer if we aren't doing it yet.
        if not match and start <= current_sel.b:
            start = current_sel.b
            end = view.size()
            match = reverse_search(view, term, start, end, flags)
            if not match:
                return
        # No luck in the whole buffer.
        elif not match:
            return
        end = match.a

    return match

def find_last_in_range(view, term, start, end, flags=0):
    found = find_in_range(view, term, start, end, flags)
    last_found = found
    while found:
        found = find_in_range(view, term, found.b, end, flags)
        if not found or found.b > end:
            break
        last_found = found if found else last_found

    return last_found

# reverse search
def reverse_search(view, term, start, end, flags=0):
    assert isinstance(start, int) or start is None
    assert isinstance(end, int) or end is None

    start = start if (start is not None) else 0
    end = end if (end is not None) else view.size()

    if start < 0 or end > view.size():
        return None

    lo_line = view.full_line(start)
    hi_line = view.full_line(end)

    while True:
        low_row, hi_row = view.rowcol(lo_line.a)[0], view.rowcol(hi_line.a)[0]
        middle_row = (low_row + hi_row) // 2

        middle_line = view.full_line(view.text_point(middle_row, 0))

        lo_region = sublime.Region(lo_line.a, middle_line.b)
        hi_region = sublime.Region(middle_line.b, min(hi_line.b, end))

        if find_in_range(view, term, hi_region.a, hi_region.b, flags):
            lo_line = view.full_line(middle_line.b)
        elif find_in_range(view, term, lo_region.a, lo_region.b, flags):
            hi_line = view.full_line(middle_line.a)
        else:
            return None

        if lo_line == hi_line:
            # we found the line we were looking for, now extract the match.
            return find_last_in_range(view, term, hi_line.a, min(hi_line.b, end), flags)

def reverse_search_by_pt(view, term, start, end, flags=0):
    assert isinstance(start, int) or start is None
    assert isinstance(end, int) or end is None

    start = start if (start is not None) else 0
    end = end if (end is not None) else view.size()

    if start < 0 or end > view.size():
        return None

    lo_line = view.full_line(start)
    hi_line = view.full_line(end)

    while True:
        low_row, hi_row = view.rowcol(lo_line.a)[0], view.rowcol(hi_line.a)[0]
        middle_row = (low_row + hi_row) // 2

        middle_line = view.full_line(view.text_point(middle_row, 0))

        lo_region = sublime.Region(lo_line.a, middle_line.b)
        hi_region = sublime.Region(middle_line.b, min(hi_line.b, end))

        if find_in_range(view, term, hi_region.a, hi_region.b, flags):
            lo_line = view.full_line(middle_line.b)
        elif find_in_range(view, term, lo_region.a, lo_region.b, flags):
            hi_line = view.full_line(middle_line.a)
        else:
            return None

        if lo_line == hi_line:
            # we found the line we were looking for, now extract the match.
            return find_last_in_range(view, term, max(hi_line.a, start), min(hi_line.b, end), flags)

# TODO: Test me.
class BufferSearchBase(sublime_plugin.TextCommand):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def calculate_flags(self):
        # TODO: Implement smartcase?
        flags = 0
        if self.view.settings().get('vintageous_magic') == False:
             flags |= sublime.LITERAL

        if self.view.settings().get('vintageous_ignorecase') == True:
            flags |= sublime.IGNORECASE

        return flags

    def build_pattern(self, query):
        return query

    def hilite(self, query):
        flags = self.calculate_flags()
        regs = self.view.find_all(self.build_pattern(query), flags)

        if not regs:
            self.view.erase_regions('vi_search')
            return

        # TODO: Re-enable this.
        # if State(self.view).settings.vi['hlsearch'] == False:
        #     return

        self.view.add_regions('vi_search', regs, 'comment', '',
                              sublime.DRAW_NO_FILL)

# TODO: Test me.
class ExactWordBufferSearchBase(BufferSearchBase):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def calculate_flags(self):
        if self.view.settings().get('vintageous_ignorecase') == True:
            return sublime.IGNORECASE
        return 0

    def get_query(self):
        # TODO: make sure we swallow any leading white space.
        query = self.view.substr(self.view.word(self.view.sel()[0].end()))
        return query

    def build_pattern(self, query):
        return r'\b{0}\b'.format(re.escape(query))
<commit_msg>refactor function<commit_after>import sublime
import sublime_plugin

import re

def find_in_range(view, term, start, end, flags=0):
    found = view.find(term, start, flags)
    if found and found.b <= end:
        return found

def find_all_in_range(view, term, start, end, flags=0):
    matches = []
    while True:
        m = find_in_range(view, term, start, end, flags)

        if m == sublime.Region(-1, -1):
            return matches

        if not m:
            return matches

        if m.end() > end or m.begin() < start:
            return matches

        matches.append(m)
        start = m.end()

def find_wrapping(view, term, start, end, flags=0, times=1):
    try:
        current_sel = view.sel()[0]
    except IndexError:
        return

    for x in range(times):
        match = find_in_range(view, term, start, end, flags)
        # make sure we wrap around the end of the buffer
        if not match:
            start = 0
            end = current_sel.a
            match = find_in_range(view, term, start, end, flags)
            if not match:
                return
        start = match.b

    return match

def reverse_find_wrapping(view, term, start, end, flags=0, times=1):
    current_sel = view.sel()[0]
    # Search wrapping around the end of the buffer.
    for x in range(times):
        match = reverse_search(view, term, start, end, flags)
        # Start searching in the lower half of the buffer if we aren't doing it yet.
        if not match and start <= current_sel.b:
            start = current_sel.b
            end = view.size()
            match = reverse_search(view, term, start, end, flags)
            if not match:
                return
        # No luck in the whole buffer.
        elif not match:
            return
        end = match.a

    return match

def find_last_in_range(view, term, start, end, flags=0):
    found = find_in_range(view, term, start, end, flags)
    last_found = found
    while found:
        found = find_in_range(view, term, found.b, end, flags)
        if not found or found.b > end:
            break
        last_found = found if found else last_found

    return last_found

# reverse search
def reverse_search(view, term, start, end, flags=0):
    assert isinstance(start, int) or start is None
    assert isinstance(end, int) or end is None

    start = start if (start is not None) else 0
    end = end if (end is not None) else view.size()

    if start < 0 or end > view.size():
        return None

    lo_line = view.full_line(start)
    hi_line = view.full_line(end)

    while True:
        low_row, hi_row = view.rowcol(lo_line.a)[0], view.rowcol(hi_line.a)[0]
        middle_row = (low_row + hi_row) // 2

        middle_line = view.full_line(view.text_point(middle_row, 0))

        lo_region = sublime.Region(lo_line.a, middle_line.b)
        hi_region = sublime.Region(middle_line.b, min(hi_line.b, end))

        if find_in_range(view, term, hi_region.a, hi_region.b, flags):
            lo_line = view.full_line(middle_line.b)
        elif find_in_range(view, term, lo_region.a, lo_region.b, flags):
            hi_line = view.full_line(middle_line.a)
        else:
            return None

        if lo_line == hi_line:
            # we found the line we were looking for, now extract the match.
            return find_last_in_range(view, term, hi_line.a, min(hi_line.b, end), flags)

def reverse_search_by_pt(view, term, start, end, flags=0):
    assert isinstance(start, int) or start is None
    assert isinstance(end, int) or end is None

    start = start if (start is not None) else 0
    end = end if (end is not None) else view.size()

    if start < 0 or end > view.size():
        return None

    lo_line = view.full_line(start)
    hi_line = view.full_line(end)

    while True:
        low_row, hi_row = view.rowcol(lo_line.a)[0], view.rowcol(hi_line.a)[0]
        middle_row = (low_row + hi_row) // 2

        middle_line = view.full_line(view.text_point(middle_row, 0))

        lo_region = sublime.Region(lo_line.a, middle_line.b)
        hi_region = sublime.Region(middle_line.b, min(hi_line.b, end))

        if find_in_range(view, term, hi_region.a, hi_region.b, flags):
            lo_line = view.full_line(middle_line.b)
        elif find_in_range(view, term, lo_region.a, lo_region.b, flags):
            hi_line = view.full_line(middle_line.a)
        else:
            return None

        if lo_line == hi_line:
            # we found the line we were looking for, now extract the match.
            return find_last_in_range(view, term, max(hi_line.a, start), min(hi_line.b, end), flags)

# TODO: Test me.
class BufferSearchBase(sublime_plugin.TextCommand):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def calculate_flags(self):
        # TODO: Implement smartcase?
        flags = 0
        if self.view.settings().get('vintageous_magic') == False:
             flags |= sublime.LITERAL

        if self.view.settings().get('vintageous_ignorecase') == True:
            flags |= sublime.IGNORECASE

        return flags

    def build_pattern(self, query):
        return query

    def hilite(self, query):
        flags = self.calculate_flags()
        regs = self.view.find_all(self.build_pattern(query), flags)

        if not regs:
            self.view.erase_regions('vi_search')
            return

        # TODO: Re-enable this.
        # if State(self.view).settings.vi['hlsearch'] == False:
        #     return

        self.view.add_regions('vi_search', regs, 'comment', '',
                              sublime.DRAW_NO_FILL)

# TODO: Test me.
class ExactWordBufferSearchBase(BufferSearchBase):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def calculate_flags(self):
        if self.view.settings().get('vintageous_ignorecase') == True:
            return sublime.IGNORECASE
        return 0

    def get_query(self):
        # TODO: make sure we swallow any leading white space.
        query = self.view.substr(self.view.word(self.view.sel()[0].end()))
        return query

    def build_pattern(self, query):
        return r'\b{0}\b'.format(re.escape(query))
<|endoftext|>


<commit_before>import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import tensorflow.contrib.keras as keras

keras.layers.GlobalAveragePooling2D()

mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

# Hyperparameter
growth_k = 12
nb_block = 3
class_num = 10
batch_size = 100

def conv_layer(input, filter, kernel, stride = [2,2], layer_name="conv") :
    with tf.name_scope(layer_name) :
        network = tf.layers.conv2d(inputs=input, filters=filter, kernel_size=kernel, strides=stride, padding='SAME')
        return network

class DenseNet() :
    def __init__(self, x, nb_blocks, filters) :
        self.nb_blocks = nb_blocks
        self.filters = filters
        self.model = self.build_model(x)

    def bottleneck_layer(self, x, scope) :
        # print(x)
        with tf.name_scope(scope) :
            x = tf.layers.batch_normalization(x)
            x = tf.nn.relu(x)
            x = conv_layer(x, filter=4 * self.filters, kernel=[1,1], layer_name=scope+'_conv1')

            x = tf.layers.batch_normalization(x)
            x = tf.nn.relu(x)
            x = conv_layer(x, filter=self.filters, kernel=[3,3], layer_name=scope+'_conv2')

            # print(x)
            return x

    def transition_layer(self, x, scope) :
        with tf.name_scope(scope) :
            x = tf.layers.batch_normalization(x)
            x = tf.nn.relu*(x)
            x = conv_layer(x, filter=self.filters, kernel=[1,1], layer_name=scope+'_conv1')
            x = tf.layers.average_pooling2d(inputs=x, pool_size=2, strides=2, padding='SAME')

            return x

    def dense_block(self, input_x, nb_layers, layer_name) :
        with tf.name_scope(layer_name) :
            layers_concat = list()
            layers_concat.append(input_x)

            x = self.bottleneck_layer(input_x, scope=layer_name + '_bottleN_' + str(0))

            layers_concat.append(x)

            for i in range(nb_layers) :
                # print(i)
                x = tf.concat(layers_concat, axis=3)
                x = self.bottleneck_layer(x, scope=layer_name + '_bottleN_' + str(i+1))
                layers_concat.append(x)

            return x

    def build_model(self, input_x) :
        x = conv_layer(input_x, filter=2 * self.filters, kernel=[7,7], layer_name='conv0')
        x = tf.layers.max_pooling2d(inputs=x, pool_size=3, strides=2, padding='SAME')

        """
        for i in range(self.nb_blocks) :
            # print(i)
            # 6 -> 12 -> 32

            x = self.dense_block(input_x=x, nb_layers=4, layer_name='dense_'+str(i))
            x = self.transition_layer(x, scope='trans_'+str(i))
        """

        x = self.dense_block(input_x=x, nb_layers=6, layer_name='dense_1')
        x = self.transition_layer(x, scope='trans_1')

        x = self.dense_block(input_x=x, nb_layers=12, layer_name='dense_2')
        x = self.transition_layer(x, scope='trans_2')

        x = self.dense_block(input_x=x, nb_layers=32, layer_name='dense_3')
        x = self.transition_layer(x, scope='trans_3')

        x = self.dense_block(input_x=x, nb_layers=32, layer_name='dense_final') # in paper, nb_layers = 32
        x = tf.nn.relu(x)
        x = tf.layers.average_pooling2d(inputs=x, pool_size=7, strides=2, padding='SAME') # pool = 7*7, global average pooling ...
        x = tf.layers.dense(inputs=x, units=class_num, name='fully_connected')

        x = tf.reshape(x, [-1,10])
        return x

x = tf.placeholder(tf.float32, shape=[None, 784])
batch_images = tf.reshape(x, [-1, 28, 28, 1])

label = tf.placeholder(tf.float32, shape=[None, 10])

logits = DenseNet(x=batch_images, nb_blocks=nb_block, filters=growth_k).model

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=label, logits=logits))
train = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(cost)

correct_prediction = tf.equal(tf.argmax(logits,1), tf.argmax(label, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

tf.summary.scalar('loss', cost)
tf.summary.scalar('accuracy', accuracy)

saver = tf.train.Saver(tf.global_variables())
training_epochs = 10

with tf.Session() as sess :
    ckpt = tf.train.get_checkpoint_state('./model')
    if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path) :
        saver.restore(sess, ckpt.model_checkpoint_path)
    else :
        sess.run(tf.global_variables_initializer())

    merged = tf.summary.merge_all()
    writer = tf.summary.FileWriter('./logs', sess.graph)
   #  writer2 = tf.summary.FileWriter('./temp', sess.graph)

    global_step = 0
    for epoch in range(2) :
        total_batch = int(mnist.train.num_examples / batch_size)

        for step in range(total_batch) :
            batch_x, batch_y = mnist.train.next_batch(batch_size)

            feed_dict = {
                x : batch_x,
                label : batch_y
            }

            _, loss = sess.run([train,cost], feed_dict=feed_dict)
            correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(label, 1))
            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
            # writer.add_summary(loss_graph, global_step=step)

            if step % 100 == 0 :
                global_step += 100
                train_summary , train_accuracy = sess.run([merged,accuracy], feed_dict=feed_dict)
                    # accuracy.eval(feed_dict=feed_dict)
                print("Step:", step, "Loss:", loss, "Training accuracy:", train_accuracy)
                writer.add_summary(train_summary, global_step=epoch)

            test_feed_dict = {
                x : mnist.test.images,
                label : mnist.test.labels
            }

        accuracy_rates = sess.run(accuracy, feed_dict=test_feed_dict)
        print('Epoch:', '%04d' % (epoch + 1), '/ Accuracy =', accuracy_rates)
        # writer.add_summary(test_summary, global_step=epoch)

    saver.save(sess=sess, save_path='./model/dense.ckpt')

<commit_msg>add Densenet<commit_after>import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

# Hyperparameter
growth_k = 12
nb_block = 3
class_num = 10
batch_size = 100

def conv_layer(input, filter, kernel, stride = [2,2], layer_name="conv") :
    with tf.name_scope(layer_name) :
        network = tf.layers.conv2d(inputs=input, filters=filter, kernel_size=kernel, strides=stride, padding='SAME')
        return network

class DenseNet() :
    def __init__(self, x, nb_blocks, filters) :
        self.nb_blocks = nb_blocks
        self.filters = filters
        self.model = self.build_model(x)

    def bottleneck_layer(self, x, scope) :
        # print(x)
        with tf.name_scope(scope) :
            x = tf.layers.batch_normalization(x)
            x = tf.nn.relu(x)
            x = conv_layer(x, filter=4 * self.filters, kernel=[1,1], layer_name=scope+'_conv1')

            x = tf.layers.batch_normalization(x)
            x = tf.nn.relu(x)
            x = conv_layer(x, filter=self.filters, kernel=[3,3], layer_name=scope+'_conv2')

            # print(x)
            return x

    def transition_layer(self, x, scope) :
        with tf.name_scope(scope) :
            x = tf.layers.batch_normalization(x)
            x = tf.nn.relu*(x)
            x = conv_layer(x, filter=self.filters, kernel=[1,1], layer_name=scope+'_conv1')
            x = tf.layers.average_pooling2d(inputs=x, pool_size=2, strides=2, padding='SAME')

            return x

    def dense_block(self, input_x, nb_layers, layer_name) :
        with tf.name_scope(layer_name) :
            layers_concat = list()
            layers_concat.append(input_x)

            x = self.bottleneck_layer(input_x, scope=layer_name + '_bottleN_' + str(0))

            layers_concat.append(x)

            for i in range(nb_layers) :
                # print(i)
                x = tf.concat(layers_concat, axis=3)
                x = self.bottleneck_layer(x, scope=layer_name + '_bottleN_' + str(i+1))
                layers_concat.append(x)

            return x

    def build_model(self, input_x) :
        x = conv_layer(input_x, filter=2 * self.filters, kernel=[7,7], layer_name='conv0')
        x = tf.layers.max_pooling2d(inputs=x, pool_size=3, strides=2, padding='SAME')

        """
        for i in range(self.nb_blocks) :
            # print(i)
            # 6 -> 12 -> 32

            x = self.dense_block(input_x=x, nb_layers=4, layer_name='dense_'+str(i))
            x = self.transition_layer(x, scope='trans_'+str(i))
        """

        x = self.dense_block(input_x=x, nb_layers=6, layer_name='dense_1')
        x = self.transition_layer(x, scope='trans_1')

        x = self.dense_block(input_x=x, nb_layers=12, layer_name='dense_2')
        x = self.transition_layer(x, scope='trans_2')

        x = self.dense_block(input_x=x, nb_layers=32, layer_name='dense_3')
        x = self.transition_layer(x, scope='trans_3')

        x = self.dense_block(input_x=x, nb_layers=32, layer_name='dense_final') # in paper, nb_layers = 32
        x = tf.nn.relu(x)
        x = tf.layers.average_pooling2d(inputs=x, pool_size=7, strides=2, padding='SAME') # pool = 7*7, global average pooling ...
        x = tf.layers.dense(inputs=x, units=class_num, name='fully_connected')

        x = tf.reshape(x, [-1,10])
        return x

x = tf.placeholder(tf.float32, shape=[None, 784])
batch_images = tf.reshape(x, [-1, 28, 28, 1])

label = tf.placeholder(tf.float32, shape=[None, 10])

logits = DenseNet(x=batch_images, nb_blocks=nb_block, filters=growth_k).model

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=label, logits=logits))
train = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(cost)

correct_prediction = tf.equal(tf.argmax(logits,1), tf.argmax(label, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

tf.summary.scalar('loss', cost)
tf.summary.scalar('accuracy', accuracy)

saver = tf.train.Saver(tf.global_variables())
training_epochs = 10

with tf.Session() as sess :
    ckpt = tf.train.get_checkpoint_state('./model')
    if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path) :
        saver.restore(sess, ckpt.model_checkpoint_path)
    else :
        sess.run(tf.global_variables_initializer())

    merged = tf.summary.merge_all()
    writer = tf.summary.FileWriter('./logs', sess.graph)
   #  writer2 = tf.summary.FileWriter('./temp', sess.graph)

    global_step = 0
    for epoch in range(2) :
        total_batch = int(mnist.train.num_examples / batch_size)

        for step in range(total_batch) :
            batch_x, batch_y = mnist.train.next_batch(batch_size)

            feed_dict = {
                x : batch_x,
                label : batch_y
            }

            _, loss = sess.run([train,cost], feed_dict=feed_dict)
            correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(label, 1))
            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
            # writer.add_summary(loss_graph, global_step=step)

            if step % 100 == 0 :
                global_step += 100
                train_summary , train_accuracy = sess.run([merged,accuracy], feed_dict=feed_dict)
                    # accuracy.eval(feed_dict=feed_dict)
                print("Step:", step, "Loss:", loss, "Training accuracy:", train_accuracy)
                writer.add_summary(train_summary, global_step=epoch)

            test_feed_dict = {
                x : mnist.test.images,
                label : mnist.test.labels
            }

        accuracy_rates = sess.run(accuracy, feed_dict=test_feed_dict)
        print('Epoch:', '%04d' % (epoch + 1), '/ Accuracy =', accuracy_rates)
        # writer.add_summary(test_summary, global_step=epoch)

    saver.save(sess=sess, save_path='./model/dense.ckpt')

<|endoftext|>