<commit_before>"""
Handles a single chunk of data (16x16x128 blocks) from a Minecraft save.
"""
from io import BytesIO
from struct import pack, unpack
import array, math

class Chunk(object):
	"""Class for representing a single chunk."""
	def __init__(self, nbt):
		chunk_data = nbt['Level']
		self.coords = chunk_data['xPos'],chunk_data['zPos']
		self.sections = [None]*16
		for section_data in chunk_data['Sections']:
			add = section_data['Add'].value if 'Add' in section_data.keys() else None
			self.sections[section_data['Y'].value] = BlockArray(section_data['Blocks'].value, add, section_data['Data'].value)

	def get_coords(self):
		"""Return the coordinates of this chunk."""
		return (self.coords[0].value,self.coords[1].value)

	def get_block(self, x, y, z):
		d,m = divmod(y, 16)
		if self.sections[d]:
			return self.sections[d].get_block(x, m, z)

	def __repr__(self):
		"""Return a representation of this Chunk."""
		return "Chunk("+str(self.coords[0])+","+str(self.coords[1])+")"

class BlockArray(object):
	"""Convenience class for dealing with a Block/data byte array."""
	def __init__(self, blocksBytes=None, addBytes=None, dataBytes=None):
		"""Create a new BlockArray, defaulting to no block or data bytes."""
		if isinstance(blocksBytes, (bytearray, array.array)):
			self.blocksList = list(blocksBytes)
		else:
			self.blocksList = [0]*32768 # Create an empty block list (32768 entries of zero (air))

		if isinstance(addBytes, (bytearray, array.array)):
			self.addList = list(addBytes)
		else:
			self.addList = [0]*16384 # Create an empty data list (32768 4-bit entries of zero make 16384 byte entries)
		if isinstance(dataBytes, (bytearray, array.array)):
			self.dataList = list(dataBytes)
		else:
			self.dataList = [0]*16384 # Create an empty data list (32768 4-bit entries of zero make 16384 byte entries)

	# Get all block entries
	def get_all_blocks(self):
		"""Return the blocks that are in this BlockArray."""
		return self.blocksList

	# Get all data entries
	def get_all_data(self):
		"""Return the data of all the blocks in this BlockArray."""
		bits = []
		for b in self.dataList:
			# The first byte of the Blocks arrays correspond
			# to the LEAST significant bits of the first byte of the Data.
			# NOT to the MOST significant bits, as you might expected.
			bits.append(b & 15) # Little end of the byte
			bits.append((b >> 4) & 15) # Big end of the byte
		return bits

	def get_all_add(self):
		"""Return the data of all the blocks in this BlockArray."""
		bits = []
		for b in self.addList:
			# The first byte of the Blocks arrays correspond
			# to the LEAST significant bits of the first byte of the Data.
			# NOT to the MOST significant bits, as you might expected.
			bits.append(b & 15) # Little end of the byte
			bits.append((b >> 4) & 15) # Big end of the byte
		return bits

	# Get all block entries and data entries as tuples
	def get_all(self):
		"""Return both blocks and data, packed together as tuples."""
		return list(zip(self.get_all_blocks(), self.get_all_add(), self.get_all_data()))

	def get_blocks_struct(self):
		"""Return a dictionary with block ids keyed to (x, y, z)."""
		cur_x = 0
		cur_y = 0
		cur_z = 0
		blocks = {}
		for block_id in self.blocksList:
			blocks[(cur_x,cur_y,cur_z)] = block_id
			cur_y += 1
			if (cur_y > 127):
				cur_y = 0
				cur_z += 1
				if (cur_z > 15):
					cur_z = 0
					cur_x += 1
		return blocks

	# Give blockList back as a byte array
	def get_blocks_byte_array(self, buffer=False):
		"""Return a list of all blocks in this chunk."""
		if buffer:
			length = len(self.blocksList)
			return BytesIO(pack(">i", length)+self.get_blocks_byte_array())
		else:
			return array.array('B', self.blocksList).tostring()

	def get_data_byte_array(self, buffer=False):
		"""Return a list of data for all blocks in this chunk."""
		if buffer:
			length = len(self.dataList)
			return BytesIO(pack(">i", length)+self.get_data_byte_array())
		else:
			return array.array('B', self.dataList).tostring()

	def generate_heightmap(self, buffer=False, as_array=False):
		"""Return a heightmap, representing the highest solid blocks in this chunk."""
		non_solids = [0, 8, 9, 10, 11, 38, 37, 32, 31]
		if buffer:
			return BytesIO(pack(">i", 256)+self.generate_heightmap()) # Length + Heightmap, ready for insertion into Chunk NBT
		else:
			bytes = []
			for z in range(16):
				for x in range(16):
					for y in range(127, -1, -1):
						offset = y + z*128 + x*128*16
						if (self.blocksList[offset] not in non_solids or y == 0):
							bytes.append(y+1)
							break
			if (as_array):
				return bytes
			else:
				return array.array('B', bytes).tostring()

	def set_blocks(self, list=None, dict=None, fill_air=False):
		"""
		Sets all blocks in this chunk, using either a list or dictionary.
		Blocks not explicitly set can be filled to air by setting fill_air to True.
		"""
		if list:
			# Inputting a list like self.blocksList
			self.blocksList = list
		elif dict:
			# Inputting a dictionary like result of self.get_blocks_struct()
			list = []
			for x in range(16):
				for z in range(16):
					for y in range(128):
						coord = x,y,z
						offset = y + z*128 + x*128*16
						if (coord in dict):
							list.append(dict[coord])
						else:
							if (self.blocksList[offset] and not fill_air):
								list.append(self.blocksList[offset])
							else:
								list.append(0) # Air
			self.blocksList = list
		else:
			# None of the above...
			return False
		return True

	def set_block(self, x,y,z, id, data=0):
		"""Sets the block a x, y, z to the specified id, and optionally data."""
		offset = y + z*128 + x*128*16
		self.blocksList[offset] = id
		if (offset % 2 == 1):
			# offset is odd
			index = (offset-1)//2
			b = self.dataList[index]
			self.dataList[index] = (b & 240) + (data & 15) # modify lower bits, leaving higher bits in place
		else:
			# offset is even
			index = offset//2
			b = self.dataList[index]
			self.dataList[index] = (b & 15) + (data << 4 & 240) # modify ligher bits, leaving lower bits in place

	# Get a given X,Y,Z or a tuple of three coordinates
	def get_block(self, x,y,z, coord=False):
		"""Return the id of the block at x, y, z."""
		"""
		Laid out like:
		(0,0,0), (0,1,0), (0,2,0) ... (0,127,0), (0,0,1), (0,1,1), (0,2,1) ... (0,127,1), (0,0,2) ... (0,127,15), (1,0,0), (1,1,0) ... (15,127,15)

		::

		  blocks = []
		  for x in range(15):
		    for z in range(15):
		      for y in range(127):
		        blocks.append(Block(x,y,z))
		"""

		offset = y*16*16 + z*16 + x if (coord == False) else coord[0] + coord[2]*16 + coord[1]*16*16
		return self.blocksList[offset]

	# Get a given X,Y,Z or a tuple of three coordinates
	def get_data(self, x,y,z, coord=False):
		"""Return the data of the block at x, y, z."""
		offset = y*16*16 + z*16 + x if (coord == False) else coord[0] + coord[2]*16 + coord[1]*16*16
		# The first byte of the Blocks arrays correspond
		# to the LEAST significant bits of the first byte of the Data.
		# NOT to the MOST significant bits, as you might expected.
		if (offset % 2 == 1):
			# offset is odd
			index = (offset-1)//2
			b = self.dataList[index]
			return b & 15 # Get little (last 4 bits) end of byte
		else:
			# offset is even
			index = offset//2
			b = self.dataList[index]
			return (b >> 4) & 15 # Get big end (first 4 bits) of byte

	def get_block_and_data(self, x,y,z, coord=False):
		"""Return the tuple of (id, data) for the block at x, y, z"""
		return (self.get_block(x,y,z,coord),self.get_data(x,y,z,coord))

<commit_msg>Fix indentation of chunk.py<commit_after>"""
Handles a single chunk of data (16x16x128 blocks) from a Minecraft save.
"""
from io import BytesIO
from struct import pack, unpack
import array, math

class Chunk(object):
    """Class for representing a single chunk."""
    def __init__(self, nbt):
        chunk_data = nbt['Level']
        self.coords = chunk_data['xPos'],chunk_data['zPos']
        self.sections = [None]*16
        for section_data in chunk_data['Sections']:
            add = section_data['Add'].value if 'Add' in section_data.keys() else None
            self.sections[section_data['Y'].value] = BlockArray(section_data['Blocks'].value, add, section_data['Data'].value)

    def get_coords(self):
        """Return the coordinates of this chunk."""
        return (self.coords[0].value,self.coords[1].value)

    def get_block(self, x, y, z):
        d,m = divmod(y, 16)
        if self.sections[d]:
            return self.sections[d].get_block(x, m, z)

    def __repr__(self):
        """Return a representation of this Chunk."""
        return "Chunk("+str(self.coords[0])+","+str(self.coords[1])+")"

class BlockArray(object):
    """Convenience class for dealing with a Block/data byte array."""
    def __init__(self, blocksBytes=None, addBytes=None, dataBytes=None):
        """Create a new BlockArray, defaulting to no block or data bytes."""
        if isinstance(blocksBytes, (bytearray, array.array)):
            self.blocksList = list(blocksBytes)
        else:
            self.blocksList = [0]*32768 # Create an empty block list (32768 entries of zero (air))

        if isinstance(addBytes, (bytearray, array.array)):
            self.addList = list(addBytes)
        else:
            self.addList = [0]*16384 # Create an empty data list (32768 4-bit entries of zero make 16384 byte entries)
        if isinstance(dataBytes, (bytearray, array.array)):
            self.dataList = list(dataBytes)
        else:
            self.dataList = [0]*16384 # Create an empty data list (32768 4-bit entries of zero make 16384 byte entries)

    # Get all block entries
    def get_all_blocks(self):
        """Return the blocks that are in this BlockArray."""
        return self.blocksList

    # Get all data entries
    def get_all_data(self):
        """Return the data of all the blocks in this BlockArray."""
        bits = []
        for b in self.dataList:
            # The first byte of the Blocks arrays correspond
            # to the LEAST significant bits of the first byte of the Data.
            # NOT to the MOST significant bits, as you might expected.
            bits.append(b & 15) # Little end of the byte
            bits.append((b >> 4) & 15) # Big end of the byte
        return bits

    def get_all_add(self):
        """Return the data of all the blocks in this BlockArray."""
        bits = []
        for b in self.addList:
            # The first byte of the Blocks arrays correspond
            # to the LEAST significant bits of the first byte of the Data.
            # NOT to the MOST significant bits, as you might expected.
            bits.append(b & 15) # Little end of the byte
            bits.append((b >> 4) & 15) # Big end of the byte
        return bits

    # Get all block entries and data entries as tuples
    def get_all(self):
        """Return both blocks and data, packed together as tuples."""
        return list(zip(self.get_all_blocks(), self.get_all_add(), self.get_all_data()))

    def get_blocks_struct(self):
        """Return a dictionary with block ids keyed to (x, y, z)."""
        cur_x = 0
        cur_y = 0
        cur_z = 0
        blocks = {}
        for block_id in self.blocksList:
            blocks[(cur_x,cur_y,cur_z)] = block_id
            cur_y += 1
            if (cur_y > 127):
                cur_y = 0
                cur_z += 1
                if (cur_z > 15):
                    cur_z = 0
                    cur_x += 1
        return blocks

    # Give blockList back as a byte array
    def get_blocks_byte_array(self, buffer=False):
        """Return a list of all blocks in this chunk."""
        if buffer:
            length = len(self.blocksList)
            return BytesIO(pack(">i", length)+self.get_blocks_byte_array())
        else:
            return array.array('B', self.blocksList).tostring()

    def get_data_byte_array(self, buffer=False):
        """Return a list of data for all blocks in this chunk."""
        if buffer:
            length = len(self.dataList)
            return BytesIO(pack(">i", length)+self.get_data_byte_array())
        else:
            return array.array('B', self.dataList).tostring()

    def generate_heightmap(self, buffer=False, as_array=False):
        """Return a heightmap, representing the highest solid blocks in this chunk."""
        non_solids = [0, 8, 9, 10, 11, 38, 37, 32, 31]
        if buffer:
            return BytesIO(pack(">i", 256)+self.generate_heightmap()) # Length + Heightmap, ready for insertion into Chunk NBT
        else:
            bytes = []
            for z in range(16):
                for x in range(16):
                    for y in range(127, -1, -1):
                        offset = y + z*128 + x*128*16
                        if (self.blocksList[offset] not in non_solids or y == 0):
                            bytes.append(y+1)
                            break
            if (as_array):
                return bytes
            else:
                return array.array('B', bytes).tostring()

    def set_blocks(self, list=None, dict=None, fill_air=False):
        """
        Sets all blocks in this chunk, using either a list or dictionary.
        Blocks not explicitly set can be filled to air by setting fill_air to True.
        """
        if list:
            # Inputting a list like self.blocksList
            self.blocksList = list
        elif dict:
            # Inputting a dictionary like result of self.get_blocks_struct()
            list = []
            for x in range(16):
                for z in range(16):
                    for y in range(128):
                        coord = x,y,z
                        offset = y + z*128 + x*128*16
                        if (coord in dict):
                            list.append(dict[coord])
                        else:
                            if (self.blocksList[offset] and not fill_air):
                                list.append(self.blocksList[offset])
                            else:
                                list.append(0) # Air
            self.blocksList = list
        else:
            # None of the above...
            return False
        return True

    def set_block(self, x,y,z, id, data=0):
        """Sets the block a x, y, z to the specified id, and optionally data."""
        offset = y + z*128 + x*128*16
        self.blocksList[offset] = id
        if (offset % 2 == 1):
            # offset is odd
            index = (offset-1)//2
            b = self.dataList[index]
            self.dataList[index] = (b & 240) + (data & 15) # modify lower bits, leaving higher bits in place
        else:
            # offset is even
            index = offset//2
            b = self.dataList[index]
            self.dataList[index] = (b & 15) + (data << 4 & 240) # modify ligher bits, leaving lower bits in place

    # Get a given X,Y,Z or a tuple of three coordinates
    def get_block(self, x,y,z, coord=False):
        """Return the id of the block at x, y, z."""
        """
        Laid out like:
        (0,0,0), (0,1,0), (0,2,0) ... (0,127,0), (0,0,1), (0,1,1), (0,2,1) ... (0,127,1), (0,0,2) ... (0,127,15), (1,0,0), (1,1,0) ... (15,127,15)

        ::

          blocks = []
          for x in range(15):
            for z in range(15):
              for y in range(127):
                blocks.append(Block(x,y,z))
        """

        offset = y*16*16 + z*16 + x if (coord == False) else coord[0] + coord[2]*16 + coord[1]*16*16
        return self.blocksList[offset]

    # Get a given X,Y,Z or a tuple of three coordinates
    def get_data(self, x,y,z, coord=False):
        """Return the data of the block at x, y, z."""
        offset = y*16*16 + z*16 + x if (coord == False) else coord[0] + coord[2]*16 + coord[1]*16*16
        # The first byte of the Blocks arrays correspond
        # to the LEAST significant bits of the first byte of the Data.
        # NOT to the MOST significant bits, as you might expected.
        if (offset % 2 == 1):
            # offset is odd
            index = (offset-1)//2
            b = self.dataList[index]
            return b & 15 # Get little (last 4 bits) end of byte
        else:
            # offset is even
            index = offset//2
            b = self.dataList[index]
            return (b >> 4) & 15 # Get big end (first 4 bits) of byte

    def get_block_and_data(self, x,y,z, coord=False):
        """Return the tuple of (id, data) for the block at x, y, z"""
        return (self.get_block(x,y,z,coord),self.get_data(x,y,z,coord))

<|endoftext|>


<commit_before># Copyright 2015 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# Copyright 2015-2016 Antonio Espinosa <antonio.espinosa@tecnativa.com>
# Copyright 2015 Javier Iniesta <javieria@antiun.com>
# Copyright 2020 Tecnativa - Manuel Calero
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    "name": "Link partners with mass-mailing",
    "version": "13.0.1.0.1",
    "author": "Tecnativa, " "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/social",
    "license": "AGPL-3",
    "category": "Marketing",
    "depends": ["mass_mailing"],
    "post_init_hook": "post_init_hook",
    "data": [
        "views/mailing_trace_view.xml",
        "views/mailing_contact_view.xml",
        "views/mailing_view.xml",
        "views/res_partner_view.xml",
        "wizard/partner_mail_list_wizard.xml",
    ],
    "installable": True,
}
<commit_msg>mass_mailing_partner 13.0.1.0.2<commit_after># Copyright 2015 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# Copyright 2015-2016 Antonio Espinosa <antonio.espinosa@tecnativa.com>
# Copyright 2015 Javier Iniesta <javieria@antiun.com>
# Copyright 2020 Tecnativa - Manuel Calero
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    "name": "Link partners with mass-mailing",
    "version": "13.0.1.0.2",
    "author": "Tecnativa, " "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/social",
    "license": "AGPL-3",
    "category": "Marketing",
    "depends": ["mass_mailing"],
    "post_init_hook": "post_init_hook",
    "data": [
        "views/mailing_trace_view.xml",
        "views/mailing_contact_view.xml",
        "views/mailing_view.xml",
        "views/res_partner_view.xml",
        "wizard/partner_mail_list_wizard.xml",
    ],
    "installable": True,
}
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Coypyright (C) 2010, University of Oxford
#
# Licensed under the MIT License.  You may obtain a copy of the License at:
#
#     http://www.opensource.org/licenses/mit-license.php
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# $Id: $

"""
Main CGI handler program for submitting data files from ADMIRAL to be stored as
an RDF Databank dataset.
"""
__author__ = "Bhavana Ananda"
__version__ = "0.1"

import cgi, sys, re, logging, os, os.path,traceback
from rdflib import URIRef
sys.path.append("..")
sys.path.append("../..")

import SubmitDatasetDetailsHandler
import SubmitDatasetUtils
import ManifestRDFUtils
import HttpSession
from MiscLib import TestUtils

siloName                 = "admiral-test" # @@FIXME
save_stdout              = sys.stdout
dcterms                  =  URIRef("http://purl.org/dc/terms/")
oxds                     =  URIRef("http://vocab.ox.ac.uk/dataset/schema#")
NamespaceDictionary      =  {
                               "dcterms"   : dcterms ,
                               "oxds"      : oxds
                            }
ZipMimeType              =  "application/zip"
FilePat                  =  re.compile("^.*$(?<!\.zip)")
Logger                   =  logging.getLogger("SubmitDatasetConfirmationHandler")
ElementCreatorUri        =  URIRef(dcterms + "creator")
ElementIdentifierUri     =  URIRef(dcterms + "identifier")
ElementTitleUri          =  URIRef(dcterms + "title")
ElementDescriptionUri    =  URIRef(dcterms + "description")
ElementUriList           =  [ElementCreatorUri, ElementIdentifierUri, ElementTitleUri, ElementDescriptionUri]
DefaultManifestName      =  "manifest.rdf"
BaseDir                  =  "/home/"
SuccessStatus            =  "ADMIRAL dataset submission to Oxford University Databank was successful"

def processDatasetSubmissionForm(formdata, outputstr):
    """
    Process form data, and print (to stdout) a new HTML page reflecting
    the outcome of the request.

    formdata    is a dictionary containing parameters from the dataset submission form
    """
    userName             =  SubmitDatasetUtils.getFormParam("user"        ,  formdata)
    userPass             =  SubmitDatasetUtils.getFormParam("pass"        ,  formdata)
    endpointhost         =  SubmitDatasetUtils.getFormParam("endpointhost",  formdata)
    basepath             =  SubmitDatasetUtils.getFormParam("basepath"    ,  formdata)
    datasetName          =  SubmitDatasetUtils.getFormParam("datId"       ,  formdata)
    title                =  SubmitDatasetUtils.getFormParam("title"       ,  formdata)
    description          =  SubmitDatasetUtils.getFormParam("description" ,  formdata)
    dirName              =  SubmitDatasetUtils.getFormParam("datDir"      ,  formdata)
    ElementValueList     =  [userName, datasetName, title, description]

    if endpointhost==None:
        endpointhost = "localhost"
    if basepath==None:
        basepath = "/admiral-test/"
    ###print("\n---- processDatasetSubmissionForm:formdata ---- \n"+repr(formdata))

    # Zip the selected Directory
    zipFileName = os.path.basename(dirName) +".zip"
    zipFilePath = "/tmp/" + zipFileName
    Logger.debug("zipFilePath = "+zipFilePath)

    if outputstr:
        sys.stdout = outputstr
    try:
        # Validate the dataset name and dataset directory fields
        validateFields(dirName, datasetName)

        # Set user credentials
        #session.setRequestUserPass(userName,userPass)

        #Create Session
        session  = HttpSession.makeHttpSession(endpointhost, basepath, userName, userPass)

        SubmitDatasetUtils.createDataset(session, siloName, datasetName)
        # Update the local manifest
        manifestFilePath     = dirName + str(os.path.sep) + DefaultManifestName
        Logger.debug("Element List = " + repr(ElementUriList))
        Logger.debug("Element Value List = " + repr(ElementValueList))
        SubmitDatasetDetailsHandler.updateMetadataInDirectoryBeforeSubmission(manifestFilePath, ElementUriList, ElementValueList)

        #Logger.debug("datasetName %s, dirName %s, zipFileName %s"%(datasetName,dirName,zipFileName))
        SubmitDatasetUtils.zipLocalDirectory(dirName, FilePat, zipFilePath)
        # Submit zip file to dataset
        SubmitDatasetUtils.submitFileToDataset(session, siloName, datasetName, zipFileName, zipFilePath, ZipMimeType, zipFileName)
        # Unzip the contents into a new dataset
        datasetUnzippedName = SubmitDatasetUtils.unzipRemoteFileToNewDataset(session, siloName, datasetName, zipFileName)
        # Redirect to the Dataset Summary page
        redirectToSubmissionSummaryPage(dirName, datasetName+"-packed", datasetUnzippedName, convertToUriString(SuccessStatus))
        return

    except SubmitDatasetUtils.SubmitDatasetError, e:
        SubmitDatasetUtils.printHTMLHeaders()
        SubmitDatasetUtils.generateErrorResponsePageFromException(e)

    except session.HTTPSessionError, e:
        SubmitDatasetUtils.printHTMLHeaders()
        SubmitDatasetUtils.generateErrorResponsePage(
            SubmitDatasetUtils.HTTP_ERROR,
            e.code, e.reason)
        SubmitDatasetUtils.printStackTrace()

    except:
        SubmitDatasetUtils.printHTMLHeaders()
        print "
Server error while processing dataset submission
"
        print "
Diagnostic stack trace follows
"
        SubmitDatasetUtils.printStackTrace()
        raise

    finally:
        print ""
        print ""
        Logger.debug("zipFilePath = "+zipFilePath)
        SubmitDatasetUtils.deleteLocalFile(zipFilePath)# Delete the local zip file after submission
        sys.stdout = save_stdout
        ###print "---- manifestFilePath "+manifestFilePath
        ###print "---- ElementValueList "+repr(ElementValueList)
    return

def validateFields(datasetDirectoryName, datasetName):
    datIDPattern = re.compile("^[a-zA-Z0-9._:-]+$")
    matchedString = datIDPattern.match(datasetName)

    if matchedString==None:
        raise SubmitDatasetUtils.SubmitDatasetError(
            SubmitDatasetUtils.INPUT_ERROR,
            None,
            "Not a valid Dataset ID: '"+datasetName+"' supplied")

    if datasetDirectoryName.endswith('/'):
        raise SubmitDatasetUtils.SubmitDatasetError(
            SubmitDatasetUtils.INPUT_ERROR,
            None,
            "Expecting no trailing '/' on directory name: '"+datasetDirectoryName+"' supplied")
    return

def convertToUriString(statusString):
    statusString = SuccessStatus.replace(" ", "%20")
    return statusString

def redirectToSubmissionSummaryPage(dirName, datasetName, datasetUnzippedName, statusText):
    print "Status: 303 Dataset submission successful"
    print "Location: SubmitDatasetSummaryHandler.py?dir=%s&id=%s&unzipid=%s&status=%s" % (dirName,datasetName,datasetUnzippedName, statusText)
    print

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    form = cgi.FieldStorage()   # Parse the query
    os.chdir("/home/")           # Base directory for admiral server data
    processDatasetSubmissionForm(form, sys.stdout)

# End.
<commit_msg>(More updates) The UI error after refactoring HTTPUtils is fixed. Modified SubmitDatasetUtils to use the session variables instead.<commit_after>#!/usr/bin/python
#
# Coypyright (C) 2010, University of Oxford
#
# Licensed under the MIT License.  You may obtain a copy of the License at:
#
#     http://www.opensource.org/licenses/mit-license.php
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# $Id: $

"""
Main CGI handler program for submitting data files from ADMIRAL to be stored as
an RDF Databank dataset.
"""
__author__ = "Bhavana Ananda"
__version__ = "0.1"

import cgi, sys, re, logging, os, os.path,traceback
from rdflib import URIRef
sys.path.append("..")
sys.path.append("../..")

import SubmitDatasetDetailsHandler
import SubmitDatasetUtils
import ManifestRDFUtils
import HttpSession
from MiscLib import TestUtils

siloName                 = "admiral-test" # @@FIXME
save_stdout              = sys.stdout
dcterms                  =  URIRef("http://purl.org/dc/terms/")
oxds                     =  URIRef("http://vocab.ox.ac.uk/dataset/schema#")
NamespaceDictionary      =  {
                               "dcterms"   : dcterms ,
                               "oxds"      : oxds
                            }
ZipMimeType              =  "application/zip"
FilePat                  =  re.compile("^.*$(?<!\.zip)")
Logger                   =  logging.getLogger("SubmitDatasetConfirmationHandler")
ElementCreatorUri        =  URIRef(dcterms + "creator")
ElementIdentifierUri     =  URIRef(dcterms + "identifier")
ElementTitleUri          =  URIRef(dcterms + "title")
ElementDescriptionUri    =  URIRef(dcterms + "description")
ElementUriList           =  [ElementCreatorUri, ElementIdentifierUri, ElementTitleUri, ElementDescriptionUri]
DefaultManifestName      =  "manifest.rdf"
BaseDir                  =  "/home/"
SuccessStatus            =  "ADMIRAL dataset submission to Oxford University Databank was successful"

def processDatasetSubmissionForm(formdata, outputstr):
    """
    Process form data, and print (to stdout) a new HTML page reflecting
    the outcome of the request.

    formdata    is a dictionary containing parameters from the dataset submission form
    """
    userName             =  SubmitDatasetUtils.getFormParam("user"        ,  formdata)
    userPass             =  SubmitDatasetUtils.getFormParam("pass"        ,  formdata)
    endpointhost         =  SubmitDatasetUtils.getFormParam("endpointhost",  formdata)
    basepath             =  SubmitDatasetUtils.getFormParam("basepath"    ,  formdata)
    datasetName          =  SubmitDatasetUtils.getFormParam("datId"       ,  formdata)
    title                =  SubmitDatasetUtils.getFormParam("title"       ,  formdata)
    description          =  SubmitDatasetUtils.getFormParam("description" ,  formdata)
    dirName              =  SubmitDatasetUtils.getFormParam("datDir"      ,  formdata)
    ElementValueList     =  [userName, datasetName, title, description]

    #if endpointhost==None:
    endpointhost = "localhost"
    #if basepath==None:
    basepath = "/admiral-test/"
    ###print("\n---- processDatasetSubmissionForm:formdata ---- \n"+repr(formdata))

    # Zip the selected Directory
    zipFileName = os.path.basename(dirName) +".zip"
    zipFilePath = "/tmp/" + zipFileName
    Logger.debug("zipFilePath = "+zipFilePath)

    if outputstr:
        sys.stdout = outputstr
    try:
        # Validate the dataset name and dataset directory fields
        validateFields(dirName, datasetName)

        # Set user credentials
        #session.setRequestUserPass(userName,userPass)

        #Create Session
        session  = HttpSession.makeHttpSession(endpointhost, basepath, userName, userPass)

        SubmitDatasetUtils.createDataset(session, siloName, datasetName)
        # Update the local manifest
        manifestFilePath     = dirName + str(os.path.sep) + DefaultManifestName
        Logger.debug("Element List = " + repr(ElementUriList))
        Logger.debug("Element Value List = " + repr(ElementValueList))
        SubmitDatasetDetailsHandler.updateMetadataInDirectoryBeforeSubmission(manifestFilePath, ElementUriList, ElementValueList)

        #Logger.debug("datasetName %s, dirName %s, zipFileName %s"%(datasetName,dirName,zipFileName))
        SubmitDatasetUtils.zipLocalDirectory(dirName, FilePat, zipFilePath)
        # Submit zip file to dataset
        SubmitDatasetUtils.submitFileToDataset(session, siloName, datasetName, zipFileName, zipFilePath, ZipMimeType, zipFileName)
        # Unzip the contents into a new dataset
        datasetUnzippedName = SubmitDatasetUtils.unzipRemoteFileToNewDataset(session, siloName, datasetName, zipFileName)
        # Redirect to the Dataset Summary page
        redirectToSubmissionSummaryPage(dirName, datasetName+"-packed", datasetUnzippedName, convertToUriString(SuccessStatus))
        return

    except SubmitDatasetUtils.SubmitDatasetError, e:
        SubmitDatasetUtils.printHTMLHeaders()
        SubmitDatasetUtils.generateErrorResponsePageFromException(e)

    except session.HTTPSessionError, e:
        SubmitDatasetUtils.printHTMLHeaders()
        SubmitDatasetUtils.generateErrorResponsePage(
            SubmitDatasetUtils.HTTP_ERROR,
            e.code, e.reason)
        SubmitDatasetUtils.printStackTrace()

    except:
        SubmitDatasetUtils.printHTMLHeaders()
        print "
Server error while processing dataset submission
"
        print "
Diagnostic stack trace follows
"
        SubmitDatasetUtils.printStackTrace()
        raise

    finally:
        print ""
        print ""
        Logger.debug("zipFilePath = "+zipFilePath)
        SubmitDatasetUtils.deleteLocalFile(zipFilePath)# Delete the local zip file after submission
        sys.stdout = save_stdout
        ###print "---- manifestFilePath "+manifestFilePath
        ###print "---- ElementValueList "+repr(ElementValueList)
    return

def validateFields(datasetDirectoryName, datasetName):
    datIDPattern = re.compile("^[a-zA-Z0-9._:-]+$")
    matchedString = datIDPattern.match(datasetName)

    if matchedString==None:
        raise SubmitDatasetUtils.SubmitDatasetError(
            SubmitDatasetUtils.INPUT_ERROR,
            None,
            "Not a valid Dataset ID: '"+datasetName+"' supplied")

    if datasetDirectoryName.endswith('/'):
        raise SubmitDatasetUtils.SubmitDatasetError(
            SubmitDatasetUtils.INPUT_ERROR,
            None,
            "Expecting no trailing '/' on directory name: '"+datasetDirectoryName+"' supplied")
    return

def convertToUriString(statusString):
    statusString = SuccessStatus.replace(" ", "%20")
    return statusString

def redirectToSubmissionSummaryPage(dirName, datasetName, datasetUnzippedName, statusText):
    print "Status: 303 Dataset submission successful"
    print "Location: SubmitDatasetSummaryHandler.py?dir=%s&id=%s&unzipid=%s&status=%s" % (dirName,datasetName,datasetUnzippedName, statusText)
    print

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    form = cgi.FieldStorage()   # Parse the query
    os.chdir("/home/")           # Base directory for admiral server data
    processDatasetSubmissionForm(form, sys.stdout)

# End.
<|endoftext|>


<commit_before># Server program
from socket import socket, AF_INET, SOCK_STREAM
from json_creator import get_message_type, JimAction, get_username
import argparse
import log_config
from socketserver import TCPServer, StreamRequestHandler, ThreadingMixIn
from storage import ServerStorage
from ast import literal_eval as str_to_dict

log = log_config.Log('server')

class MessageHandler(StreamRequestHandler):
    def handle(self):
        print('Client connected - {}:{}'.format(*self.client_address))
        self.server.add_client(self)
        while True:
            data = self.rfile.readline().strip()
            client_message = data.decode('utf-8')
            print(client_message)
            client_message_dict = str_to_dict(client_message)
            self.server.parse_client_message(self, client_message_dict)
            self.server.write_to_all_clients(client_message)
        self.server.remove_client(self)
        print('Client disconnected - {}:{}'.format(*self.client_address))

class Server(ThreadingMixIn, TCPServer):
    allow_reuse_address = True
    max_children = 10

    def __init__(self, server_address, request_handler_class):
        super().__init__(server_address, request_handler_class, True)
        self.clients = set()
        self.storage = ServerStorage()

    def add_client(self, client):
        self.clients.add(client)

    def remove_client(self, client):
        self.clients.remove(client)

    def write_to_client(self, client, message):
        print('Sent:', message)
        client.wfile.write((message + '\n').encode('utf-8'))

    def write_to_all_clients(self, message):
        for client in self.clients:
            self.write_to_client(client, message)

    def parse_client_message(self, client, message):
        ip = client.client_address[0]
        message_type = get_message_type(message)
        if message_type == JimAction.presence:
            self.register_client_connect(get_username(message), ip)
        elif message_type == JimAction.get_all_contacts:
            self._send_all_contacts(client)

    def register_client_connect(self, username, ip):
        self.storage.save_client_connect(username, ip)

    def _send_all_contacts(self, client):
        all_contacts = self.storage.get_all_contacts()
        contacts = []
        for contact in all_contacts:
            contacts.append(contact)
        contacts_count = len(contacts)
        self.write_to_client(client, 'Start List')
        self.write_to_client(client, str(contacts_count))
        for contact in contacts:
            self.write_to_client(client, contact)
        self.write_to_client(client, 'End List')

@log
def get_server_port(**main_args):
    if __name__ == "__main__":
        # Parse script arguments
        arg_parser = argparse.ArgumentParser()
        arg_parser.add_argument('-p')
        arg_parser.add_argument('-a')
        arguments = arg_parser.parse_args()
        server_port = 7777 if (arguments.p is None) else int(arguments.p)
    else:
        # Parse main() arguments
        server_port = main_args.get('p', 7777)
    return server_port

@log
def get_ips_to_listen(**main_args):
    if __name__ == "__main__":
        # Parse script arguments
        arg_parser = argparse.ArgumentParser()
        arg_parser.add_argument('-p')
        arg_parser.add_argument('-a')
        arguments = arg_parser.parse_args()
        ip_to_listen = '' if (arguments.a is None) else str(arguments.a)
    else:
        # Parse main() arguments
        ip_to_listen = main_args.get('a', '')
    return ip_to_listen

@log
def main(**main_args):
    server_port = get_server_port(**main_args)
    ip_to_listen = get_ips_to_listen(**main_args)

    server = Server(('', server_port), MessageHandler)
    server.serve_forever()

if __name__ == "__main__":
    main()
<commit_msg>Fix server crash on unexpected client disconnect<commit_after># Server program
from socket import socket, AF_INET, SOCK_STREAM
from json_creator import \
    get_message_type, \
    JimAction, \
    get_username, \
    get_adding_friend_name
import argparse
import log_config
from socketserver import ThreadingTCPServer, StreamRequestHandler
from storage import ServerStorage
from ast import literal_eval as str_to_dict

# log = log_config.Log('server')

class MessageHandler(StreamRequestHandler):
    def handle(self):
        print('Client connected - {}:{}'.format(*self.client_address))
        self.server.add_client(self)

        while True:
            try:
                data = self.rfile.readline()
            except:
                break
            if not data:
                break
            else:
                data = data.strip()
                client_message = data.decode('utf-8')
                self.server.messages.append((self, client_message))
                print(client_message)
                client_message_dict = str_to_dict(client_message)
                self.server.parse_client_message(self, client_message_dict)

        print('Client disconnected - {}:{}'.format(*self.client_address))
        self.server.remove_client(self)

class Server(ThreadingTCPServer):
    allow_reuse_address = True
    max_children = 10

    def __init__(self, server_address, request_handler_class):
        super().__init__(server_address, request_handler_class, True)
        self.clients = set()
        self.storage = ServerStorage()
        self.messages = []

    def add_client(self, client):
        self.clients.add(client)

    def remove_client(self, client):
        self.clients.remove(client)

    def write_to_client(self, client, message):
        print('Sent:', message)
        client.wfile.write((message + '\n').encode('utf-8'))

    def write_to_all_clients(self, message, sender_client):
        for client in self.clients:
            if client is sender_client:
                continue
            self.write_to_client(client, message)

    def parse_client_message(self, client, message):
        ip = client.client_address[0]

        message_type = get_message_type(message)
        if message_type == JimAction.presence:
            self.register_client_connect(get_username(message), ip)

        elif message_type == JimAction.get_all_contacts:
            self._send_all_contacts(client)

        elif message_type == JimAction.add_friend:
            friend = get_adding_friend_name(message)
            self._add_client_friend(client, friend)

        elif message_type == JimAction.msg:
            self.write_to_all_clients(message, client)

    def register_client_connect(self, username, ip):
        self.storage.save_client_connect(username, ip)

    def _send_all_contacts(self, client):
        all_contacts = self.storage.get_all_contacts()
        contacts = []
        for contact in all_contacts:
            contacts.append(contact)
        contacts_count = len(contacts)
        self.write_to_client(client, 'Start List')
        self.write_to_client(client, str(contacts_count))
        for contact in contacts:
            self.write_to_client(client, contact)
        self.write_to_client(client, 'End List')

    def _add_client_friend(self, friend_name):
        friend_id = self.storage.add_new_friend(new_friend, gets_friend)

def get_server_port(**main_args):
    if __name__ == "__main__":
        # Parse script arguments
        arg_parser = argparse.ArgumentParser()
        arg_parser.add_argument('-p')
        arg_parser.add_argument('-a')
        arguments = arg_parser.parse_args()
        server_port = 7777 if (arguments.p is None) else int(arguments.p)
    else:
        # Parse main() arguments
        server_port = main_args.get('p', 7777)
    return server_port

def get_ips_to_listen(**main_args):
    if __name__ == "__main__":
        # Parse script arguments
        arg_parser = argparse.ArgumentParser()
        arg_parser.add_argument('-p')
        arg_parser.add_argument('-a')
        arguments = arg_parser.parse_args()
        ip_to_listen = '' if (arguments.a is None) else str(arguments.a)
    else:
        # Parse main() arguments
        ip_to_listen = main_args.get('a', '')
    return ip_to_listen

def main(**main_args):
    server_port = get_server_port(**main_args)
    ip_to_listen = get_ips_to_listen(**main_args)
    server = Server(('', server_port), MessageHandler)
    server.serve_forever()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import sys
import shutil
import platform
import subprocess
from operator import eq

frame = "======================================================="

def cmdstr(cmd):
    try:
        ret = str(subprocess.check_output(cmd, shell=True))
        if platform.system() == "Windows":
            ret = ret[2:-5]
        else:
            ret = ret[2:-3]
    except:
        print("[!] error on running " + cmd)
        ret = ""
        return ret

    print("cmdstr=" + ret)
    return ret

python3 = ""
def branch(command):
    if command == "help":
        return help()
    elif command == "history":
        return history()
    elif command == "clean":
        return clean()
    elif command == "commit":
        return commit()
    elif command == "build":
        return build()
    elif command == "run":
        arg = None if len(sys.argv) < 3 else sys.argv[2]
        return run(arg)
    elif command == "doc":
        return doc()
    elif command == "pubdoc":
        return _publishDoc()

    print(command + " is unknown.")
    return -1

def _cleanIntermediates():
    print("removing intermediate outputs...", end=" ")
    os.system("rm -rf " + cwd + "/xml")
    os.system("rm -rf " + cwd + "/*.tmp")
    os.system("git config --unset user.name") # remove local config only
    os.system("git config --unset user.email")
    print("done.")

def doc():
    # Idea from Travis Gockel.
    global cwd, python3

    _cleanIntermediates()
    os.system("rm -rf " + cwd + "/html")

    # standby gh-pages repo:
    print("cloning gh-pages branch...", end=" ")
    res = os.system("git clone -b gh-pages --depth 5 git@github.com:kniz/worldlang.git --single-branch " + cwd + "/html")
    if res != 0:
        print("fail to clone gh-pages repo.")
        _cleanIntermediates()
        return -1
    print("done.")
    os.system("git rm -rf " + cwd + "/html")

    # build doxygen + m.css:
    print("generating docs using doxygen...", end=" ")
    res = os.system(python3 + " " + cwd + "/m.css/doxygen/dox2html5.py " + cwd + "/Doxyfile")
    if res != 0:
        print("fail to run m.css doxy parser.")
        _cleanIntermediates()
        return -1
    print("done.")

def _publishDoc():
    # pushing on gh-pages:
    origin = cmdstr("git rev-parse --verify HEAD")
    print("origin=" + str(origin))
    os.chdir(cwd + "/html")
    os.system("git add .")
    os.system("git config user.name \"autodocbot\"") # put on local config.
    os.system("git config user.email \"knizofficial@gmail.com\"")
    res = os.system("git commit -m \"The our poor little Autobot \(❍ᴥ❍ʋ)/ generated docs for " + origin + ", clitter-clatter.\"")
    if res != 0:
        print("fail to commit on gh-pages.")
        print("it seems that nothing changed.")
        _cleanIntermediates()
        return -1
    res = os.system("git push origin gh-pages")
    if res != 0:
        print("fail to push on gh-pages")
        _cleanIntermediates()
        return -1
    os.chdir(cwd)

    _cleanIntermediates()
    return 0

def run(arg):
    if arg is None:
        print("build world and run one of next followings...")
        print("\t * unittests")
        return -1

    if arg == "unittests":
        result = build()
        if result:
            return result
        return _ut()

# currently this application only supports window and linux.
def isWindow():
    return platform.system().find("Windows") >= 0

# def _beautify():
#     print("beautify source codes...")
#     for path, dirs, files in os.walk("..\\"):
#         for file in files:
#             file_path = os.path.join(path, file)
#             ext = os.path.splitext(file_path)[1]
#             if  ext == ".cpp" or ext == ".c" or ext == ".hpp" or ext == ".h" or \
#                 ext == ".inl":
#                 # change encoding:
#                 # beautify
#                 print("\t" + file)
#                 os.system("astyle --style=world " + file_path)

def _createMakefiles():
    print("")
    generator = "MinGW Makefiles" if isWindow() else "Unix Makefiles"
    print("generating makefiles as " + generator + "...")

    res = os.system("cmake . -G \"" + generator + "\"")
    if res != 0:
        return -1

# World uses BuildInfo at RELEASE.md
# and builder.py drives to all world libraries and make it sync all build-info including
# doc releasing.
ver_major = 0
ver_minor = 0
ver_fix = 0
ver_name = ""
ver_buildcnt = 0
def _extractBuildInfo(): # from RELEASE.md at root directory.
    global cwd, ver_major, ver_minor, ver_fix, ver_name, ver_buildcnt
    path = cwd + "/../RELEASE.md"

    fp = open(path, "r")
    while True:
        line = fp.readline()
        if not line: break
        if line[:4] in "## v":
            minor_head_n = line.find('.', 5) + 1
            ver_major = int(line[4:minor_head_n-1])
            ver_minor = int(line[minor_head_n: minor_head_n+1])
            ver_name_n = line.find(' ', minor_head_n+1)+1
            ver_fix_str = line[minor_head_n+1: ver_name_n]
            if ver_fix_str in "" or ver_fix_str in " ":
                ver_fix = 0
            else:
                ver_fix = int(ver_fix_str)
            ver_name = line[ver_name_n: len(line)-6-1-1]
            break

    fp.close()
updated = False
def _updateLine(lines, n, trg, basestr):
    global updated
    idx = len(basestr)-1
    value = int(lines[n][idx:len(lines[n])-2])
    if value != trg:
        lines[n] = basestr + str(trg) + ")\n"
        print(lines[n])
        updated = True

def _updateLineString(lines, n, trg, basestr):
    global updated
    idx = len(basestr)-1
    value = lines[n][idx:len(lines[n])-2]
    if eq(value, trg):
        lines[n] = basestr + str(trg) + ")\n"
        print(lines[n])
        updated = True

def _injectBuildInfo():
    global cwd, ver_major, ver_minor, ver_fix, ver_name, ver_buildcnt
    path = cwd + "/CMakeLists.txt"

    print("updating buildinfo on CMakeLists.txt...", end=" ")
    global updated
    updated = False;
    fp = open(path, "r")
    lines = fp.readlines()
    for n in range(0, len(lines)):
        line = lines[n]
        if line[:17] in "set(VERSION_MAJOR":
            _updateLine(lines, n, ver_major, "set(VERSION_MAJOR ")
        elif line[:17] in "set(VERSION_MINOR":
            _updateLine(lines, n, ver_minor, "set(VERSION_MINOR ")
        elif line[:15] in "set(VERSION_FIX":
            _updateLine(lines, n, ver_fix, "set(VERSION_FIX ")
        elif line[:16] in "set(VERSION_NAME":
            _updateLineString(lines, n, ver_name, "set(VERSION_NAME ");
    fp.close()

    if updated == False:
        print("skip")
        return

    fp = open(path, "w")
    fp.write("".join(lines))
    fp.close()

def _incBuildCnt():
    global cwd
    path = cwd + "/CMakeLists.txt"

    print("Increase Build count...", end=" ")
    fp = open(path, "r")
    lines = fp.readlines()
    buildcnt = 0
    for n in range(0, len(lines)):
        line = lines[n]
        if line[:15] in "set(BUILD_COUNT":
            buildcnt = int(line[16:len(line)-2])
            lines[n] = "set(BUILD_COUNT " + str(buildcnt+1) + ")\n"
            break
    fp.close()

    fp = open(path, "w")
    fp.write("".join(lines))
    fp.close()
    print("done")

def _make():
    print("")
    make_option = "-j4 -s"  # j4 -> 4 multithread.
                            # s ->  don't print command.
    print("making " + make_option + "...", end=" ")
    if isWindow():
        os.system("mingw32-make -v")
        result = os.system("mingw32-make " + make_option)
        if result != 0:
            print("failed")
            return -1
    else:
        os.system("make -v")
        result = os.system("make " + make_option)
        if result != 0:
            print("failed")
            return -1
    print("done")

def build():
    #_beautify()
    _injectBuildInfo()
    if _createMakefiles():
        return -1
    if _make():
        return -1
    if _ut():
        return -1
    _incBuildCnt()
    return 0

def _ut():
    print("")
    print("let's initiate unit tests...", end=" ")
    global cwd
    res = os.system(cwd + "/unittests")
    if res == 0:
        print("done")
    return res

def commit():
    return 0

def _extractPythonVersion(verstr):
    return float(verstr[7:10])

def checkDependencies():
    global python3
    print("")
    print ("checking dependencies...", end=" ")
    simple_depencies = ["git", "cmake", "java"]

    for e in simple_depencies:
        if not shutil.which(e):
            print("[!] " + e + " is NOT installed!")
            return -1

    if isWindow():
        if not shutil.which("mingw32-make"):
            print("[!] mingw32-make on Windows is NOT installed!")
            return -1
    elif not shutil.which("make"):
        print("[!] make for linux is NOT installed!")
        return -1

    if _extractPythonVersion(cmdstr(python3 + " --version")) < 3.6:
        print("[!] requires python over v3.6")
        return -1
    print("done")

def version():
    global ver_name, ver_major, ver_minor, ver_fix, cwd, python3
    print("Builder. Support-utility for building World " + ver_name + " v" + str(ver_major) + "." + str(ver_minor) + str(ver_fix))
    print("Copyrights (c) kniz, 2009-2018")
    print(frame)
    print("")
    print("* use python version at " + python3 + " and its version is " + cmdstr(python3 + " --version"))
    print("* building directory is " + cwd)

def help():
    print("Usage: builder.py <command> <arg1> <arg2> ...")
    print("")
    print("command list:")
    print("\t * help")
    print("\t * history")
    print("\t * clean\tclear all cache files of cmake outputs.")
    print("\t * build")
    print("\t * run\t\tbuild + run one of predefined programs.")

def clean():
    print("Clearing next following files...")
    global cwd
    _clean(cwd)
    _cleanIntermediates()
    os.system("rm -rf " + cwd + "/html")
    print("was removed successfully.")

def _clean(directory):
    for path, dirs, files in os.walk(directory):

        for file in files:
            file_path = os.path.join(path, file)
            ext = os.path.splitext(file_path)[1]
            if  ext == ".sln" or ext == ".vcproj" or ext == ".vcxproj" or ext == ".cmake" or \
                file == "CMakeCache.txt" or ext == ".stamp" or ext == ".depend" or file == "Makefile" or \
                ext == ".exe" or ext == ".dll" or ext == ".log" or ext == ".a" or ext == ".so":
                print("\t * " + file_path)
                os.remove(file_path)

        for dir in dirs:
            if dir == "CMakeFiles" or dir == "tmp" or dir == "generated":
                abs_dir = os.path.join(path, dir)
                print("\t * " + abs_dir)
                shutil.rmtree(abs_dir)
cwd = ""

def _where(name):
    cmd = ""
    prefix = ""
    if platform.system() == "Linux":
        cmd = "which"
    elif platform.system() == "Windows":
        cmd = "where"
        prefix = "\""
    else:
        print("[!] " + platform.system() + " unsupported.")
        return ""

    print("cmd=" + cmd + ", name=" + name)
    ret = prefix + cmdstr(cmd + " " + name) + prefix
    print(ret)
    return ret

def _extractEnv():
    global python3
    if "PYTHON" in os.environ:
        python3 = os.environ["PYTHON"]
    else:
        python3 = _where("python3")
        print("python3=" + python3)
        if not python3:
            print("no!")
            python3 = _where("python")
        print("python3=" + python3)
        return python3 == ""

def _init():
    global cwd
    cwd=os.path.dirname(os.path.realpath(sys.argv[0]))
    _extractBuildInfo()
    return _extractEnv()

def main():
    if _init():
        return -1
    version()

    os.chdir(cwd)
    if checkDependencies():
        print("\n[!] This program needs following softwares to be fully functional.")
        return -1

    if len(sys.argv) == 1:
        help()
    else:
        return branch(sys.argv[1])

    print("")
    return 0

ret = main()
if ret != 0:
    print("[!] ends with " + str(ret) + " exit code.")
sys.exit(ret)
<commit_msg>- [builder] builder got F/C if it fails to call "python --version"<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import sys
import shutil
import platform
import subprocess
from operator import eq

frame = "======================================================="

def cmdstr(cmd):
    try:
        ret = str(subprocess.check_output(cmd, shell=True))
        if platform.system() == "Windows":
            ret = ret[2:-5]
        else:
            ret = ret[2:-3]
    except:
        print("[!] error on running " + cmd)
        ret = ""
        return ret

    print(ret + "= call(" + cmd + ")")
    return ret

python3 = ""
def branch(command):
    if command == "help":
        return help()
    elif command == "history":
        return history()
    elif command == "clean":
        return clean()
    elif command == "commit":
        return commit()
    elif command == "build":
        return build()
    elif command == "run":
        arg = None if len(sys.argv) < 3 else sys.argv[2]
        return run(arg)
    elif command == "doc":
        return doc()
    elif command == "pubdoc":
        return _publishDoc()

    print(command + " is unknown.")
    return -1

def _cleanIntermediates():
    print("removing intermediate outputs...", end=" ")
    os.system("rm -rf " + cwd + "/xml")
    os.system("rm -rf " + cwd + "/*.tmp")
    os.system("git config --unset user.name") # remove local config only
    os.system("git config --unset user.email")
    print("done.")

def doc():
    # Idea from Travis Gockel.
    global cwd, python3

    _cleanIntermediates()
    os.system("rm -rf " + cwd + "/html")

    # standby gh-pages repo:
    print("cloning gh-pages branch...", end=" ")
    res = os.system("git clone -b gh-pages --depth 5 git@github.com:kniz/worldlang.git --single-branch " + cwd + "/html")
    if res != 0:
        print("fail to clone gh-pages repo.")
        _cleanIntermediates()
        return -1
    print("done.")
    os.system("git rm -rf " + cwd + "/html")

    # build doxygen + m.css:
    print("generating docs using doxygen...", end=" ")
    res = os.system(python3 + " " + cwd + "/m.css/doxygen/dox2html5.py " + cwd + "/Doxyfile")
    if res != 0:
        print("fail to run m.css doxy parser.")
        _cleanIntermediates()
        return -1
    print("done.")

def _publishDoc():
    # pushing on gh-pages:
    origin = cmdstr("git rev-parse --verify HEAD")
    print("origin=" + str(origin))
    os.chdir(cwd + "/html")
    os.system("git add .")
    os.system("git config user.name \"autodocbot\"") # put on local config.
    os.system("git config user.email \"knizofficial@gmail.com\"")
    res = os.system("git commit -m \"The our poor little Autobot \(❍ᴥ❍ʋ)/ generated docs for " + origin + ", clitter-clatter.\"")
    if res != 0:
        print("fail to commit on gh-pages.")
        print("it seems that nothing changed.")
        _cleanIntermediates()
        return -1
    res = os.system("git push origin gh-pages")
    if res != 0:
        print("fail to push on gh-pages")
        _cleanIntermediates()
        return -1
    os.chdir(cwd)

    _cleanIntermediates()
    return 0

def run(arg):
    if arg is None:
        print("build world and run one of next followings...")
        print("\t * unittests")
        return -1

    if arg == "unittests":
        result = build()
        if result:
            return result
        return _ut()

# currently this application only supports window and linux.
def isWindow():
    return platform.system().find("Windows") >= 0

# def _beautify():
#     print("beautify source codes...")
#     for path, dirs, files in os.walk("..\\"):
#         for file in files:
#             file_path = os.path.join(path, file)
#             ext = os.path.splitext(file_path)[1]
#             if  ext == ".cpp" or ext == ".c" or ext == ".hpp" or ext == ".h" or \
#                 ext == ".inl":
#                 # change encoding:
#                 # beautify
#                 print("\t" + file)
#                 os.system("astyle --style=world " + file_path)

def _createMakefiles():
    print("")
    generator = "MinGW Makefiles" if isWindow() else "Unix Makefiles"
    print("generating makefiles as " + generator + "...")

    res = os.system("cmake . -G \"" + generator + "\"")
    if res != 0:
        return -1

# World uses BuildInfo at RELEASE.md
# and builder.py drives to all world libraries and make it sync all build-info including
# doc releasing.
ver_major = 0
ver_minor = 0
ver_fix = 0
ver_name = ""
ver_buildcnt = 0
def _extractBuildInfo(): # from RELEASE.md at root directory.
    global cwd, ver_major, ver_minor, ver_fix, ver_name, ver_buildcnt
    path = cwd + "/../RELEASE.md"

    fp = open(path, "r")
    while True:
        line = fp.readline()
        if not line: break
        if line[:4] in "## v":
            minor_head_n = line.find('.', 5) + 1
            ver_major = int(line[4:minor_head_n-1])
            ver_minor = int(line[minor_head_n: minor_head_n+1])
            ver_name_n = line.find(' ', minor_head_n+1)+1
            ver_fix_str = line[minor_head_n+1: ver_name_n]
            if ver_fix_str in "" or ver_fix_str in " ":
                ver_fix = 0
            else:
                ver_fix = int(ver_fix_str)
            ver_name = line[ver_name_n: len(line)-6-1-1]
            break

    fp.close()
updated = False
def _updateLine(lines, n, trg, basestr):
    global updated
    idx = len(basestr)-1
    value = int(lines[n][idx:len(lines[n])-2])
    if value != trg:
        lines[n] = basestr + str(trg) + ")\n"
        print(lines[n])
        updated = True

def _updateLineString(lines, n, trg, basestr):
    global updated
    idx = len(basestr)-1
    value = lines[n][idx:len(lines[n])-2]
    if eq(value, trg):
        lines[n] = basestr + str(trg) + ")\n"
        print(lines[n])
        updated = True

def _injectBuildInfo():
    global cwd, ver_major, ver_minor, ver_fix, ver_name, ver_buildcnt
    path = cwd + "/CMakeLists.txt"

    print("updating buildinfo on CMakeLists.txt...", end=" ")
    global updated
    updated = False;
    fp = open(path, "r")
    lines = fp.readlines()
    for n in range(0, len(lines)):
        line = lines[n]
        if line[:17] in "set(VERSION_MAJOR":
            _updateLine(lines, n, ver_major, "set(VERSION_MAJOR ")
        elif line[:17] in "set(VERSION_MINOR":
            _updateLine(lines, n, ver_minor, "set(VERSION_MINOR ")
        elif line[:15] in "set(VERSION_FIX":
            _updateLine(lines, n, ver_fix, "set(VERSION_FIX ")
        elif line[:16] in "set(VERSION_NAME":
            _updateLineString(lines, n, ver_name, "set(VERSION_NAME ");
    fp.close()

    if updated == False:
        print("skip")
        return

    fp = open(path, "w")
    fp.write("".join(lines))
    fp.close()

def _incBuildCnt():
    global cwd
    path = cwd + "/CMakeLists.txt"

    print("Increase Build count...", end=" ")
    fp = open(path, "r")
    lines = fp.readlines()
    buildcnt = 0
    for n in range(0, len(lines)):
        line = lines[n]
        if line[:15] in "set(BUILD_COUNT":
            buildcnt = int(line[16:len(line)-2])
            lines[n] = "set(BUILD_COUNT " + str(buildcnt+1) + ")\n"
            break
    fp.close()

    fp = open(path, "w")
    fp.write("".join(lines))
    fp.close()
    print("done")

def _make():
    print("")
    make_option = "-j4 -s"  # j4 -> 4 multithread.
                            # s ->  don't print command.
    print("making " + make_option + "...", end=" ")
    if isWindow():
        os.system("mingw32-make -v")
        result = os.system("mingw32-make " + make_option)
        if result != 0:
            print("failed")
            return -1
    else:
        os.system("make -v")
        result = os.system("make " + make_option)
        if result != 0:
            print("failed")
            return -1
    print("done")

def build():
    #_beautify()
    _injectBuildInfo()
    if _createMakefiles():
        return -1
    if _make():
        return -1
    if _ut():
        return -1
    _incBuildCnt()
    return 0

def _ut():
    print("")
    print("let's initiate unit tests...", end=" ")
    global cwd
    res = os.system(cwd + "/unittests")
    if res == 0:
        print("done")
    return res

def commit():
    return 0

def _extractPythonVersion(verstr):
    if not verstr:
        print("couldn't get version of python.")
        return 0.0
    return float(verstr[7:10])

def checkDependencies():
    global python3
    print("")
    print ("checking dependencies...", end=" ")
    simple_depencies = ["git", "cmake", "java"]

    for e in simple_depencies:
        if not shutil.which(e):
            print("[!] " + e + " is NOT installed!")
            return -1

    if isWindow():
        if not shutil.which("mingw32-make"):
            print("[!] mingw32-make on Windows is NOT installed!")
            return -1
    elif not shutil.which("make"):
        print("[!] make for linux is NOT installed!")
        return -1

    if _extractPythonVersion(cmdstr(python3 + " --version")) < 3.6:
        print("[!] requires python over v3.6")
        return -1
    print("done")

def version():
    global ver_name, ver_major, ver_minor, ver_fix, cwd, python3
    print("Builder. Support-utility for building World " + ver_name + " v" + str(ver_major) + "." + str(ver_minor) + str(ver_fix))
    print("Copyrights (c) kniz, 2009-2018")
    print(frame)
    print("")
    print("* use python version at " + python3 + " and its version is " + cmdstr(python3 + " --version"))
    print("* building directory is " + cwd)

def help():
    print("Usage: builder.py <command> <arg1> <arg2> ...")
    print("")
    print("command list:")
    print("\t * help")
    print("\t * history")
    print("\t * clean\tclear all cache files of cmake outputs.")
    print("\t * build")
    print("\t * run\t\tbuild + run one of predefined programs.")

def clean():
    print("Clearing next following files...")
    global cwd
    _clean(cwd)
    _cleanIntermediates()
    os.system("rm -rf " + cwd + "/html")
    print("was removed successfully.")

def _clean(directory):
    for path, dirs, files in os.walk(directory):

        for file in files:
            file_path = os.path.join(path, file)
            ext = os.path.splitext(file_path)[1]
            if  ext == ".sln" or ext == ".vcproj" or ext == ".vcxproj" or ext == ".cmake" or \
                file == "CMakeCache.txt" or ext == ".stamp" or ext == ".depend" or file == "Makefile" or \
                ext == ".exe" or ext == ".dll" or ext == ".log" or ext == ".a" or ext == ".so":
                print("\t * " + file_path)
                os.remove(file_path)

        for dir in dirs:
            if dir == "CMakeFiles" or dir == "tmp" or dir == "generated":
                abs_dir = os.path.join(path, dir)
                print("\t * " + abs_dir)
                shutil.rmtree(abs_dir)
cwd = ""

def _where(name):
    cmd = ""
    prefix = ""
    if platform.system() == "Linux":
        cmd = "which"
    elif platform.system() == "Windows":
        cmd = "where"
        prefix = "\""
    else:
        print("[!] " + platform.system() + " unsupported.")
        return ""

    print("cmd=" + cmd + ", name=" + name)
    ret = prefix + cmdstr(cmd + " " + name) + prefix
    print(ret)
    return ret

def _extractEnv():
    global python3
    if "PYTHON" in os.environ:
        python3 = os.environ["PYTHON"]
    else:
        python3 = _where("python3")
        print("python3=" + python3)
        if not python3:
            print("no!")
            python3 = _where("python")
        print("python3=" + python3)
        return python3 == ""

def _init():
    global cwd
    cwd=os.path.dirname(os.path.realpath(sys.argv[0]))
    _extractBuildInfo()
    return _extractEnv()

def main():
    if _init():
        return -1
    version()

    os.chdir(cwd)
    if checkDependencies():
        print("\n[!] This program needs following softwares to be fully functional.")
        return -1

    if len(sys.argv) == 1:
        help()
    else:
        return branch(sys.argv[1])

    print("")
    return 0

ret = main()
if ret != 0:
    print("[!] ends with " + str(ret) + " exit code.")
sys.exit(ret)
<|endoftext|>


<commit_before>"""Shared acceptance test functions."""
from time import sleep

from splinter.driver.webdriver import WebDriverElement

from adhocracy_core.testing import god_login
from adhocracy_core.testing import god_password
from adhocracy_core.testing import annotator_password
from adhocracy_core.testing import annotator_login

def wait(condition, step=0.1, max_steps=10) -> bool:
    """Wait for a condition to become true."""
    for i in range(max_steps - 1):
        if condition():
            return True
        else:
            sleep(step)
    return condition()

def login(browser, name_or_email, password,
          expect_success=True,
          visit_root=True):
    """Login user with name and password."""
    if is_logged_in(browser):
        return
    login_url = browser.app_url + 'login'
    browser.visit(login_url)
    fill_input(browser, '.login [name="nameOrEmail"]', name_or_email)
    fill_input(browser, '.login [name="password"]', password)
    click_button(browser, '.login [type="submit"]')
    if expect_success:
        browser.wait_for_condition(is_logged_in, 20)
    if visit_root:
        browser.visit(browser.root_url)

def login_god(browser):
    """Login god user."""
    login(browser, god_login, god_password)

def login_annotator(browser):
    """Login god user."""
    login(browser, annotator_login, annotator_password)

def logout(browser):
    """Logout user."""
    if is_logged_in(browser):
        click_button(browser, '.user-indicator-logout')
        browser.wait_for_condition(is_logged_out, 30)

def is_logged_in(browser):
    """Check if user is logged in."""
    return browser.browser.is_element_present_by_css('.user-indicator-logout')

def is_logged_out(browser):
    """Check if user is logged out."""
    return browser.browser.is_element_not_present_by_css(
        '.user-indicator-logout')

def fill_input(browser, css_selector, value):
    """Find `css_selector` and fill value."""
    element = browser.browser.find_by_css(css_selector).first
    element.fill(value)

def click_button(browser, css_selector):
    """Find `css_selector` and click."""
    element = browser.browser.find_by_css(css_selector).first
    element.click()

def title_is_in_listing(listing, title: str) -> bool:
    """Check that a listing element with text == `title` exists."""
    for element in listing.find_by_css('.listing-element'):
        wait(lambda: element.text, max_steps=5)
        if element.text == title:
            return True

def get_listing_create_form(listing) -> WebDriverElement:
    """Open and return the create form of a listing."""
    wait(lambda: listing.find_by_css('.navbar .button'))
    button = listing.find_by_css('.navbar .button').first
    wait(lambda: button.visible)
    button.click()
    return listing.find_by_css('.listing-create-form').first

def get_column_listing(browser, column_name: str) -> WebDriverElement:
    """Return the listing in the content column ."""
    column = browser.browser.find_by_css('.moving-column-' + column_name)
    listing = column.first.find_by_css('.listing')
    return listing

def get_list_element(listing, text, descendant=None, max_steps=20):
    """Return list element with text == `text`."""
    for element in listing.find_by_css('.listing-element'):
        wait(lambda: element.text, max_steps=max_steps)
        if descendant is None:
            element_text = element.text
        else:
            element_text = element.find_by_css(descendant).first.text
        if element_text == text:
            return element
<commit_msg>Fix acceptance tests<commit_after>"""Shared acceptance test functions."""
from time import sleep

from splinter.driver.webdriver import WebDriverElement

from adhocracy_core.testing import god_login
from adhocracy_core.testing import god_password
from adhocracy_core.testing import annotator_password
from adhocracy_core.testing import annotator_login

def wait(condition, step=0.1, max_steps=10) -> bool:
    """Wait for a condition to become true."""
    for i in range(max_steps - 1):
        if condition():
            return True
        else:
            sleep(step)
    return condition()

def login(browser, name_or_email, password,
          expect_success=True,
          visit_root=True):
    """Login user with name and password."""
    if is_logged_in(browser):
        return
    login_url = browser.app_url + 'login'
    browser.visit(login_url)
    fill_input(browser, '.login [name="nameOrEmail"]', name_or_email)
    fill_input(browser, '.login [name="password"]', password)
    click_button(browser, '.login [type="submit"]')
    if expect_success:
        browser.wait_for_condition(is_logged_in, 20)
    if visit_root:
        browser.visit(browser.root_url)

def login_god(browser):
    """Login god user."""
    login(browser, god_login, god_password)

def login_annotator(browser):
    """Login god user."""
    login(browser, annotator_login, annotator_password)

def logout(browser):
    """Logout user."""
    if is_logged_in(browser):
        click_button(browser, '.user-indicator-logout')
        browser.wait_for_condition(is_logged_out, 30)

def is_logged_in(browser):
    """Check if user is logged in."""
    return browser.browser.is_element_present_by_css('.user-indicator-logout')

def is_logged_out(browser):
    """Check if user is logged out."""
    return browser.browser.is_element_not_present_by_css(
        '.user-indicator-logout')

def fill_input(browser, css_selector, value):
    """Find `css_selector` and fill value."""
    element = browser.browser.find_by_css(css_selector).first
    element.fill(value)

def click_button(browser, css_selector):
    """Find `css_selector` and click."""
    element = browser.browser.find_by_css(css_selector).first
    element.click()

def title_is_in_listing(listing, title: str) -> bool:
    """Check that a listing element with text == `title` exists."""
    for element in listing.find_by_css('.listing-element'):
        wait(lambda: element.text, max_steps=5)
        if element.text == title:
            return True

def get_listing_create_form(listing) -> WebDriverElement:
    """Open and return the create form of a listing."""
    return listing.find_by_css('.listing-create-form').first

def get_column_listing(browser, column_name: str) -> WebDriverElement:
    """Return the listing in the content column ."""
    column = browser.browser.find_by_css('.moving-column-' + column_name)
    listing = column.first.find_by_css('.listing')
    return listing

def get_list_element(listing, text, descendant=None, max_steps=20):
    """Return list element with text == `text`."""
    for element in listing.find_by_css('.listing-element'):
        wait(lambda: element.text, max_steps=max_steps)
        if descendant is None:
            element_text = element.text
        else:
            element_text = element.find_by_css(descendant).first.text
        if element_text == text:
            return element
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import sys
import shutil
import platform
import subprocess
from operator import eq

frame = "======================================================="

def cmdstr(cmd):
    try:
        ret = str(subprocess.check_output(cmd, shell=True))
        if platform.system() == "Windows":
            ret = ret[2:-5]
        else:
            ret = ret[2:-3]
    except:
        print("[!] error on running " + cmd)
        ret = ""
        return ret

    print(ret + "= call(" + cmd + ")")
    return ret

python3 = ""
def branch(command):
    if command == "help":
        return help()
    elif command == "history":
        return history()
    elif command == "clean":
        return clean()
    elif command == "commit":
        return commit()
    elif command == "build":
        return build()
    elif command == "run":
        arg = None if len(sys.argv) < 3 else sys.argv[2]
        return run(arg)
    elif command == "doc":
        return doc()
    elif command == "pubdoc":
        return _publishDoc()

    print(command + " is unknown.")
    return -1

def _cleanIntermediates():
    print("removing intermediate outputs...", end=" ")
    os.system("rm -rf " + cwd + "/xml")
    os.system("rm -rf " + cwd + "/*.tmp")
    os.system("git config --unset user.name") # remove local config only
    os.system("git config --unset user.email")
    print("done.")

def doc():
    # Idea from Travis Gockel.
    global cwd, python3, externalDir

    _cleanIntermediates()
    os.system("rm -rf " + cwd + "/html")

    # standby gh-pages repo:
    print("cloning gh-pages branch...", end=" ")
    res = os.system("git clone -b gh-pages --depth 5 git@github.com:kniz/worldlang.git --single-branch " + cwd + "/html")
    if res != 0:
        print("fail to clone gh-pages repo.")
        _cleanIntermediates()
        return -1
    print("done.")
    os.system("git rm -rf " + cwd + "/html")

    # build doxygen + m.css:
    print("generating docs using doxygen...", end=" ")
    res = os.system(python3 + " " + externalDir + "/m.css/doxygen/dox2html5.py " + cwd + "/Doxyfile")
    if res != 0:
        print("fail to run m.css doxy parser.")
        _cleanIntermediates()
        return -1
    print("done.")

def _publishDoc():
    # pushing on gh-pages:
    origin = cmdstr("git rev-parse --verify HEAD")
    print("origin=" + str(origin))
    os.chdir(cwd + "/html")
    os.system("git add .")
    os.system("git config user.name \"autodocbot\"") # put on local config.
    os.system("git config user.email \"knizofficial@gmail.com\"")
    res = os.system("git commit -m \"The our poor little Autobot \(❍ᴥ❍ʋ)/ generated docs for " + origin + ", clitter-clatter.\"")
    if res != 0:
        print("fail to commit on gh-pages.")
        print("it seems that nothing changed.")
        _cleanIntermediates()
        return 0
    res = os.system("git push origin gh-pages")
    if res != 0:
        print("fail to push on gh-pages")
        _cleanIntermediates()
        return -1
    os.chdir(cwd)

    _cleanIntermediates()
    return 0

def run(arg):
    if arg is None:
        print("build world and run one of next followings...")
        print("\t * unittests")
        return -1

    if arg == "unittests":
        result = build()
        if result:
            return result
        return _ut()

# currently this application only supports window and linux.
def isWindow():
    return platform.system().find("Windows") >= 0

# def _beautify():
#     print("beautify source codes...")
#     for path, dirs, files in os.walk("..\\"):
#         for file in files:
#             file_path = os.path.join(path, file)
#             ext = os.path.splitext(file_path)[1]
#             if  ext == ".cpp" or ext == ".c" or ext == ".hpp" or ext == ".h" or \
#                 ext == ".inl":
#                 # change encoding:
#                 # beautify
#                 print("\t" + file)
#                 os.system("astyle --style=world " + file_path)

def _createMakefiles():
    print("")
    generator = "MinGW Makefiles" if isWindow() else "Unix Makefiles"
    print("generating makefiles as " + generator + "...")

    res = os.system("cmake . -G \"" + generator + "\"")
    if res != 0:
        return -1

# World uses BuildInfo at RELEASE.md
# and builder.py drives to all world libraries and make it sync all build-info including
# doc releasing.
ver_major = 0
ver_minor = 0
ver_fix = 0
ver_name = ""
ver_buildcnt = 0
def _extractBuildInfo(): # from RELEASE.md at root directory.
    global cwd, ver_major, ver_minor, ver_fix, ver_name, ver_buildcnt
    path = cwd + "/../RELEASE.md"

    fp = open(path, "r")
    while True:
        line = fp.readline()
        if not line: break
        if line[:4] in "## v":
            minor_head_n = line.find('.', 5) + 1
            ver_major = int(line[4:minor_head_n-1])
            ver_minor = int(line[minor_head_n: minor_head_n+1])
            ver_name_n = line.find(' ', minor_head_n+1)+1
            ver_fix_str = line[minor_head_n+1: ver_name_n]
            if ver_fix_str in "" or ver_fix_str in " ":
                ver_fix = 0
            else:
                ver_fix = int(ver_fix_str)
            ver_name = line[ver_name_n: len(line)-6-1-1]
            break

    fp.close()
updated = False
def _updateLine(lines, n, trg, basestr):
    global updated
    idx = len(basestr)-1
    value = int(lines[n][idx:len(lines[n])-2])
    if value != trg:
        lines[n] = basestr + str(trg) + ")\n"
        print(lines[n])
        updated = True

def _updateLineString(lines, n, trg, basestr):
    global updated
    idx = len(basestr)-1
    value = lines[n][idx:len(lines[n])-2]
    if eq(value, trg):
        lines[n] = basestr + str(trg) + ")\n"
        print(lines[n])
        updated = True

def _injectBuildInfo():
    global cwd, ver_major, ver_minor, ver_fix, ver_name, ver_buildcnt
    path = cwd + "/CMakeLists.txt"

    print("updating buildinfo on CMakeLists.txt...", end=" ")
    global updated
    updated = False;
    fp = open(path, "r")
    lines = fp.readlines()
    for n in range(0, len(lines)):
        line = lines[n]
        if line[:17] in "set(VERSION_MAJOR":
            _updateLine(lines, n, ver_major, "set(VERSION_MAJOR ")
        elif line[:17] in "set(VERSION_MINOR":
            _updateLine(lines, n, ver_minor, "set(VERSION_MINOR ")
        elif line[:15] in "set(VERSION_FIX":
            _updateLine(lines, n, ver_fix, "set(VERSION_FIX ")
        elif line[:16] in "set(VERSION_NAME":
            _updateLineString(lines, n, ver_name, "set(VERSION_NAME ");
    fp.close()

    if updated == False:
        print("skip")
        return

    fp = open(path, "w")
    fp.write("".join(lines))
    fp.close()

def _incBuildCnt():
    global cwd
    path = cwd + "/CMakeLists.txt"

    print("Increase Build count...", end=" ")
    fp = open(path, "r")
    lines = fp.readlines()
    buildcnt = 0
    for n in range(0, len(lines)):
        line = lines[n]
        if line[:15] in "set(BUILD_COUNT":
            buildcnt = int(line[16:len(line)-2])
            lines[n] = "set(BUILD_COUNT " + str(buildcnt+1) + ")\n"
            break
    fp.close()

    fp = open(path, "w")
    fp.write("".join(lines))
    fp.close()
    print("done")

def _make():
    print("")
    make_option = "-j4 -s"  # j4 -> 4 multithread.
                            # s ->  don't print command.
    print("making " + make_option + "...", end=" ")
    if isWindow():
        os.system("mingw32-make -v")
        result = os.system("mingw32-make " + make_option)
        if result != 0:
            print("failed")
            return -1
    else:
        os.system("make -v")
        result = os.system("make " + make_option)
        if result != 0:
            print("failed")
            return -1
    print("done")

def build():
    #_beautify()
    _injectBuildInfo()
    if _createMakefiles():
        return -1
    if _make():
        return -1
    if _ut():
        return -1
    _incBuildCnt()
    return 0

def _ut():
    print("")
    print("let's initiate unit tests...")
    global cwd, binDir

    files = os.listdir(binDir)
    ret = 0
    for file in files:
        if len(file) < 4 or file[-4:] != "Test":
            continue
        res = os.system(binDir + "/" + file)
        if res != 0: ret = res
    return ret

def commit():
    return 0

def _extractPythonVersion(verstr):
    if not verstr:
        print("couldn't get version of python.")
        return 0.0
    return float(verstr[7:10])

def checkDependencies():
    global python3
    print("")
    print ("checking dependencies...", end=" ")
    simple_depencies = ["git", "cmake", "java", "g++", "doxygen"]

    for e in simple_depencies:
        if not shutil.which(e):
            print("[!] " + e + " is NOT installed!")
            return -1

    if isWindow():
        if not shutil.which("mingw32-make"):
            print("[!] mingw32-make on Windows is NOT installed!")
            return -1
    elif not shutil.which("make"):
        print("[!] make for linux is NOT installed!")
        return -1

    if _extractPythonVersion(cmdstr(python3 + " --version")) < 3.6:
        print("[!] requires python over v3.6")
        return -1
    print("done")

def version():
    global ver_name, ver_major, ver_minor, ver_fix, cwd, python3
    print("")
    print("Builder. Support-utility for building World " + ver_name + " v" + str(ver_major) + "." + str(ver_minor) + str(ver_fix))
    print("Copyrights (c) kniz, 2009-2018")
    print(frame)
    print("")
    print("* use python version at " + python3 + " and its version is " + cmdstr(python3 + " --version"))
    print("* building directory is " + cwd)

def help():
    print("Usage: builder.py <command> <arg1> <arg2> ...")
    print("")
    print("command list:")
    print("\t * help")
    print("\t * history")
    print("\t * clean\tclear all cache files of cmake outputs.")
    print("\t * build")
    print("\t * run\t\tbuild + run one of predefined programs.")

def clean():
    print("Clearing next following files...")
    global cwd, binDir, externalDir
    _clean(cwd)
    _cleanIntermediates()
    _cleanDir(binDir)
    _cleanDir(externalDir)
    _cleanDir(cwd + "/mods")
    os.system("rm -rf " + cwd + "/html")
    print("was removed successfully.")

def _clean(directory):
    for path, dirs, files in os.walk(directory):

        for file in files:
            file_path = os.path.join(path, file)
            ext = os.path.splitext(file_path)[1]
            if  ext == ".sln" or ext == ".vcproj" or ext == ".vcxproj" or ext == ".cmake" or \
                file == "CMakeCache.txt" or ext == ".stamp" or ext == ".depend" or file == "Makefile" or \
                ext == ".exe" or ext == ".dll" or ext == ".log" or ext == ".a" or ext == ".so":
                print("\t * " + file_path)
                os.remove(file_path)

        for dir in dirs:
            if dir == "CMakeFiles" or dir == "tmp" or dir == "generated":
                abs_dir = os.path.join(path, dir)
                print("\t * " + abs_dir)
                _cleanDir(abs_dir)

def _cleanDir(dir):
    if os.path.isdir(dir) == False: return
    shutil.rmtree(dir)

def _where(name):
    cmd = ""
    prefix = ""
    if platform.system() == "Linux" or platform.system() == "Darwin":
        cmd = "which"
    elif platform.system() == "Windows":
        cmd = "where"
        prefix = "\""
    else:
        print("[!] " + platform.system() + " unsupported.")
        return ""

    print("cmd=" + cmd + ", name=" + name)
    ret = prefix + cmdstr(cmd + " " + name) + prefix
    print(ret)
    return ret

def _extractEnv():
    global python3
    if "PYTHON" in os.environ:
        python3 = os.environ["PYTHON"]
    else:
        python3 = _where("python3")
        if not python3:
            print("no!")
            python3 = _where("python")
        return python3 == ""
cwd = ""
wrdDir = ""
binDir = ""
externalDir = ""

def _init():
    global cwd, wrdDir, binDir, externalDir
    cwd = os.path.dirname(os.path.realpath(sys.argv[0]))
    wrdDir = cwd + "/.."
    binDir = wrdDir + "/bin"
    externalDir = wrdDir + "/external"

    _extractBuildInfo()
    return _extractEnv()

def main():
    if _init():
        return -1
    version()

    os.chdir(cwd)
    if checkDependencies():
        print("\n[!] This program needs following softwares to be fully functional.")
        return -1

    if len(sys.argv) == 1:
        help()
    else:
        return branch(sys.argv[1])

    print("")
    return 0

ret = main()
if ret != 0:
    print("[!] ends with " + str(ret) + " exit code.")
sys.exit(ret)
<commit_msg>- [builder] "clean" command removes external folder too.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import sys
import shutil
import platform
import subprocess
from operator import eq

frame = "======================================================="

def cmdstr(cmd):
    try:
        ret = str(subprocess.check_output(cmd, shell=True))
        if platform.system() == "Windows":
            ret = ret[2:-5]
        else:
            ret = ret[2:-3]
    except:
        print("[!] error on running " + cmd)
        ret = ""
        return ret

    print(ret + "= call(" + cmd + ")")
    return ret

python3 = ""
def branch(command):
    if command == "help":
        return help()
    elif command == "history":
        return history()
    elif command == "clean":
        return clean()
    elif command == "commit":
        return commit()
    elif command == "build":
        return build()
    elif command == "run":
        arg = None if len(sys.argv) < 3 else sys.argv[2]
        return run(arg)
    elif command == "doc":
        return doc()
    elif command == "pubdoc":
        return _publishDoc()

    print(command + " is unknown.")
    return -1

def _cleanIntermediates():
    print("removing intermediate outputs...", end=" ")
    os.system("rm -rf " + cwd + "/xml")
    os.system("rm -rf " + cwd + "/*.tmp")
    os.system("git config --unset user.name") # remove local config only
    os.system("git config --unset user.email")
    print("done.")

def doc():
    # Idea from Travis Gockel.
    global cwd, python3, externalDir

    _cleanIntermediates()
    os.system("rm -rf " + cwd + "/html")

    # standby gh-pages repo:
    print("cloning gh-pages branch...", end=" ")
    res = os.system("git clone -b gh-pages --depth 5 git@github.com:kniz/worldlang.git --single-branch " + cwd + "/html")
    if res != 0:
        print("fail to clone gh-pages repo.")
        _cleanIntermediates()
        return -1
    print("done.")
    os.system("git rm -rf " + cwd + "/html")

    # build doxygen + m.css:
    print("generating docs using doxygen...", end=" ")
    res = os.system(python3 + " " + externalDir + "/m.css/doxygen/dox2html5.py " + cwd + "/Doxyfile")
    if res != 0:
        print("fail to run m.css doxy parser.")
        _cleanIntermediates()
        return -1
    print("done.")

def _publishDoc():
    # pushing on gh-pages:
    origin = cmdstr("git rev-parse --verify HEAD")
    print("origin=" + str(origin))
    os.chdir(cwd + "/html")
    os.system("git add .")
    os.system("git config user.name \"autodocbot\"") # put on local config.
    os.system("git config user.email \"knizofficial@gmail.com\"")
    res = os.system("git commit -m \"The our poor little Autobot \(❍ᴥ❍ʋ)/ generated docs for " + origin + ", clitter-clatter.\"")
    if res != 0:
        print("fail to commit on gh-pages.")
        print("it seems that nothing changed.")
        _cleanIntermediates()
        return 0
    res = os.system("git push origin gh-pages")
    if res != 0:
        print("fail to push on gh-pages")
        _cleanIntermediates()
        return -1
    os.chdir(cwd)

    _cleanIntermediates()
    return 0

def run(arg):
    if arg is None:
        print("build world and run one of next followings...")
        print("\t * unittests")
        return -1

    if arg == "unittests":
        result = build()
        if result:
            return result
        return _ut()

# currently this application only supports window and linux.
def isWindow():
    return platform.system().find("Windows") >= 0

# def _beautify():
#     print("beautify source codes...")
#     for path, dirs, files in os.walk("..\\"):
#         for file in files:
#             file_path = os.path.join(path, file)
#             ext = os.path.splitext(file_path)[1]
#             if  ext == ".cpp" or ext == ".c" or ext == ".hpp" or ext == ".h" or \
#                 ext == ".inl":
#                 # change encoding:
#                 # beautify
#                 print("\t" + file)
#                 os.system("astyle --style=world " + file_path)

def _createMakefiles():
    print("")
    generator = "MinGW Makefiles" if isWindow() else "Unix Makefiles"
    print("generating makefiles as " + generator + "...")

    res = os.system("cmake . -G \"" + generator + "\"")
    if res != 0:
        return -1

# World uses BuildInfo at RELEASE.md
# and builder.py drives to all world libraries and make it sync all build-info including
# doc releasing.
ver_major = 0
ver_minor = 0
ver_fix = 0
ver_name = ""
ver_buildcnt = 0
def _extractBuildInfo(): # from RELEASE.md at root directory.
    global cwd, ver_major, ver_minor, ver_fix, ver_name, ver_buildcnt
    path = cwd + "/../RELEASE.md"

    fp = open(path, "r")
    while True:
        line = fp.readline()
        if not line: break
        if line[:4] in "## v":
            minor_head_n = line.find('.', 5) + 1
            ver_major = int(line[4:minor_head_n-1])
            ver_minor = int(line[minor_head_n: minor_head_n+1])
            ver_name_n = line.find(' ', minor_head_n+1)+1
            ver_fix_str = line[minor_head_n+1: ver_name_n]
            if ver_fix_str in "" or ver_fix_str in " ":
                ver_fix = 0
            else:
                ver_fix = int(ver_fix_str)
            ver_name = line[ver_name_n: len(line)-6-1-1]
            break

    fp.close()
updated = False
def _updateLine(lines, n, trg, basestr):
    global updated
    idx = len(basestr)-1
    value = int(lines[n][idx:len(lines[n])-2])
    if value != trg:
        lines[n] = basestr + str(trg) + ")\n"
        print(lines[n])
        updated = True

def _updateLineString(lines, n, trg, basestr):
    global updated
    idx = len(basestr)-1
    value = lines[n][idx:len(lines[n])-2]
    if eq(value, trg):
        lines[n] = basestr + str(trg) + ")\n"
        print(lines[n])
        updated = True

def _injectBuildInfo():
    global cwd, ver_major, ver_minor, ver_fix, ver_name, ver_buildcnt
    path = cwd + "/CMakeLists.txt"

    print("updating buildinfo on CMakeLists.txt...", end=" ")
    global updated
    updated = False;
    fp = open(path, "r")
    lines = fp.readlines()
    for n in range(0, len(lines)):
        line = lines[n]
        if line[:17] in "set(VERSION_MAJOR":
            _updateLine(lines, n, ver_major, "set(VERSION_MAJOR ")
        elif line[:17] in "set(VERSION_MINOR":
            _updateLine(lines, n, ver_minor, "set(VERSION_MINOR ")
        elif line[:15] in "set(VERSION_FIX":
            _updateLine(lines, n, ver_fix, "set(VERSION_FIX ")
        elif line[:16] in "set(VERSION_NAME":
            _updateLineString(lines, n, ver_name, "set(VERSION_NAME ");
    fp.close()

    if updated == False:
        print("skip")
        return

    fp = open(path, "w")
    fp.write("".join(lines))
    fp.close()

def _incBuildCnt():
    global cwd
    path = cwd + "/CMakeLists.txt"

    print("Increase Build count...", end=" ")
    fp = open(path, "r")
    lines = fp.readlines()
    buildcnt = 0
    for n in range(0, len(lines)):
        line = lines[n]
        if line[:15] in "set(BUILD_COUNT":
            buildcnt = int(line[16:len(line)-2])
            lines[n] = "set(BUILD_COUNT " + str(buildcnt+1) + ")\n"
            break
    fp.close()

    fp = open(path, "w")
    fp.write("".join(lines))
    fp.close()
    print("done")

def _make():
    print("")
    make_option = "-j4 -s"  # j4 -> 4 multithread.
                            # s ->  don't print command.
    print("making " + make_option + "...", end=" ")
    if isWindow():
        os.system("mingw32-make -v")
        result = os.system("mingw32-make " + make_option)
        if result != 0:
            print("failed")
            return -1
    else:
        os.system("make -v")
        result = os.system("make " + make_option)
        if result != 0:
            print("failed")
            return -1
    print("done")

def build():
    #_beautify()
    _injectBuildInfo()
    if _createMakefiles():
        return -1
    if _make():
        return -1
    if _ut():
        return -1
    _incBuildCnt()
    return 0

def _ut():
    print("")
    print("let's initiate unit tests...")
    global cwd, binDir

    files = os.listdir(binDir)
    ret = 0
    for file in files:
        if len(file) < 4 or file[-4:] != "Test":
            continue
        res = os.system(binDir + "/" + file)
        if res != 0: ret = res
    return ret

def commit():
    return 0

def _extractPythonVersion(verstr):
    if not verstr:
        print("couldn't get version of python.")
        return 0.0
    return float(verstr[7:10])

def checkDependencies():
    global python3
    print("")
    print ("checking dependencies...", end=" ")
    simple_depencies = ["git", "cmake", "java", "g++", "doxygen"]

    for e in simple_depencies:
        if not shutil.which(e):
            print("[!] " + e + " is NOT installed!")
            return -1

    if isWindow():
        if not shutil.which("mingw32-make"):
            print("[!] mingw32-make on Windows is NOT installed!")
            return -1
    elif not shutil.which("make"):
        print("[!] make for linux is NOT installed!")
        return -1

    if _extractPythonVersion(cmdstr(python3 + " --version")) < 3.6:
        print("[!] requires python over v3.6")
        return -1
    print("done")

def version():
    global ver_name, ver_major, ver_minor, ver_fix, cwd, python3
    print("")
    print("Builder. Support-utility for building World " + ver_name + " v" + str(ver_major) + "." + str(ver_minor) + str(ver_fix))
    print("Copyrights (c) kniz, 2009-2018")
    print(frame)
    print("")
    print("* use python version at " + python3 + " and its version is " + cmdstr(python3 + " --version"))
    print("* building directory is " + cwd)

def help():
    print("Usage: builder.py <command> <arg1> <arg2> ...")
    print("")
    print("command list:")
    print("\t * help")
    print("\t * history")
    print("\t * clean\tclear all cache files of cmake outputs.")
    print("\t * build")
    print("\t * run\t\tbuild + run one of predefined programs.")

def clean():
    print("Clearing next following files...")
    global cwd, binDir, externalDir
    _clean(cwd)
    _cleanIntermediates()
    _cleanDir(binDir)
    _cleanDir(cwd + "/mods")
    os.system("rm -rf " + cwd + "/html")
    print("was removed successfully.")

def _clean(directory):
    for path, dirs, files in os.walk(directory):

        for file in files:
            file_path = os.path.join(path, file)
            ext = os.path.splitext(file_path)[1]
            if  ext == ".sln" or ext == ".vcproj" or ext == ".vcxproj" or ext == ".cmake" or \
                file == "CMakeCache.txt" or ext == ".stamp" or ext == ".depend" or file == "Makefile" or \
                ext == ".exe" or ext == ".dll" or ext == ".log" or ext == ".a" or ext == ".so":
                print("\t * " + file_path)
                os.remove(file_path)

        for dir in dirs:
            if dir == "CMakeFiles" or dir == "tmp" or dir == "generated":
                abs_dir = os.path.join(path, dir)
                print("\t * " + abs_dir)
                _cleanDir(abs_dir)

def _cleanDir(dir):
    if os.path.isdir(dir) == False: return
    shutil.rmtree(dir)

def _where(name):
    cmd = ""
    prefix = ""
    if platform.system() == "Linux" or platform.system() == "Darwin":
        cmd = "which"
    elif platform.system() == "Windows":
        cmd = "where"
        prefix = "\""
    else:
        print("[!] " + platform.system() + " unsupported.")
        return ""

    print("cmd=" + cmd + ", name=" + name)
    ret = prefix + cmdstr(cmd + " " + name) + prefix
    print(ret)
    return ret

def _extractEnv():
    global python3
    if "PYTHON" in os.environ:
        python3 = os.environ["PYTHON"]
    else:
        python3 = _where("python3")
        if not python3:
            print("no!")
            python3 = _where("python")
        return python3 == ""
cwd = ""
wrdDir = ""
binDir = ""
externalDir = ""

def _init():
    global cwd, wrdDir, binDir, externalDir
    cwd = os.path.dirname(os.path.realpath(sys.argv[0]))
    wrdDir = cwd + "/.."
    binDir = wrdDir + "/bin"
    externalDir = wrdDir + "/external"

    _extractBuildInfo()
    return _extractEnv()

def main():
    if _init():
        return -1
    version()

    os.chdir(cwd)
    if checkDependencies():
        print("\n[!] This program needs following softwares to be fully functional.")
        return -1

    if len(sys.argv) == 1:
        help()
    else:
        return branch(sys.argv[1])

    print("")
    return 0

ret = main()
if ret != 0:
    print("[!] ends with " + str(ret) + " exit code.")
sys.exit(ret)
<|endoftext|>