<commit_before>from . import types

class PrettyPrinter(object):

	def __init__(self):
		self.buf = []

	def newline(self):
		self.buf.append('\n')

	def write(self, data):
		assert isinstance(data, str), data
		self.buf.append(data)

	def writeline(self, ln):
		if not ln: return
		self.write(ln + '\n')

	def visit(self, node):
		if isinstance(node, types.base):
			self.type(node)
		else:
			getattr(self, node.__class__.__name__)(node)

	def build(self, name, fun):

		name = '.'.join(name) if isinstance(name, tuple) else name
		self.start = False
		self.write('def %s(' % name)
		for i, node in enumerate(fun.args):
			self.visit(node)
			if i < len(fun.args) - 1:
				self.write(', ')

		self.write(')')
		if fun.rtype is not None:
			self.write(' -> ')
			self.visit(fun.rtype)

		self.write(':')
		self.newline()

		for i, bl in fun.flow.blocks.iteritems():
			self.writeline('  %2i: [%s]' % (i, bl.anno))
			for sid, step in enumerate(bl.steps):
				self.write(' {%02i} ' % sid)
				self.visit(step)
				self.newline()

		return ''.join(self.buf)

	def type(self, node):
		self.write(node.name)

	def anno(self, node):

		if node.type is None and not getattr(node, 'escapes', False):
			return

		if isinstance(node.type, (types.function, types.Type)):
			return

		self.write(' [')
		if node.type is not None:
			self.visit(node.type)

		if node.type is not None and getattr(node, 'escapes', False):
			self.write(':')

		if getattr(node, 'escapes', False):
			self.write('E')

		self.write(']')

	def NoneVal(self, node):
		self.write('NoneVal')
		self.anno(node)

	def Name(self, node):
		self.write(node.name)
		self.anno(node)

	def Bool(self, node):
		self.write(str(node.val))
		self.anno(node)

	def Int(self, node):
		self.write(str(node.val))
		self.anno(node)

	def Float(self, node):
		self.write(str(node.val))
		self.anno(node)

	def String(self, node):
		self.write(repr(node.val))
		self.anno(node)

	def binary(self, op, node):
		self.write(op + ' ')
		self.visit(node.left)
		self.write(' ')
		self.visit(node.right)

	def As(self, node):
		self.binary('As', node)

	def Not(self, node):
		self.write('Not ')
		self.visit(node.value)

	def And(self, node):
		self.binary('And', node)

	def Or(self, node):
		self.binary('Or', node)

	def Is(self, node):
		self.binary('Is', node)

	def NE(self, node):
		self.binary('NE', node)

	def GT(self, node):
		self.binary('GT', node)

	def LT(self, node):
		self.binary('LT', node)

	def EQ(self, node):
		self.binary('EQ', node)

	def Add(self, node):
		self.binary('Add', node)

	def Sub(self, node):
		self.binary('Sub', node)

	def Mod(self, node):
		self.binary('Mod', node)

	def Mul(self, node):
		self.binary('Mul', node)

	def Div(self, node):
		self.binary('Div', node)

	def BWAnd(self, node):
		self.binary('BWAnd', node)

	def BWOr(self, node):
		self.binary('BWOr', node)

	def BWXor(self, node):
		self.binary('BWXor', node)

	def Pass(self, node):
		self.write('Pass')

	def Init(self, node):
		self.write('Init ')
		self.visit(node.type)

	def Ref(self, node):
		self.write('&')
		self.visit(node.value)

	def Owner(self, node):
		self.write('$')
		self.visit(node.value)

	def Opt(self, node):
		self.visit(node.value)
		self.write('?')

	def Argument(self, node):
		self.write(node.name.name)
		self.anno(node)

	def Assign(self, node):
		self.visit(node.left)
		self.write(' = ')
		self.visit(node.right)

	def Call(self, node):
		self.visit(node.name)
		self.write('(')
		for i, arg in enumerate(node.args):
			self.visit(arg)
			if i < len(node.args) - 1:
				self.write(', ')
		self.write(')')
		self.anno(node)
		if node.callbr is not None:
			self.write(' => %s, %s' % node.callbr)

	def Return(self, node):
		self.write('Return')
		if node.value is not None:
			self.write(' ')
			self.visit(node.value)

	def Yield(self, node):
		self.write('Yield')
		if node.value is not None:
			self.write(' ')
			self.visit(node.value)

	def Raise(self, node):
		self.write('Raise ')
		self.visit(node.value)

	def CondBranch(self, node):
		self.write('CondBranch ')
		self.visit(node.cond)
		self.write(' ? %s : %s' % (node.tg1, node.tg2))

	def Branch(self, node):
		self.write('Branch %s' % node.label)

	def Attrib(self, node):
		self.visit(node.obj)
		self.write(' . ')
		self.write(node.attrib)
		self.anno(node)

	def SetAttr(self, node):
		self.Attrib(node)

	def Elem(self, node):
		self.write('Elem(')
		self.visit(node.obj)
		self.write(', ')
		self.visit(node.key)
		self.write(')')
		self.anno(node)

	def Phi(self, node):
		self.write('Phi ')
		self.write('%s:' % node.left[0])
		self.visit(node.left[1])
		self.write(', ')
		self.write('%i:' % node.right[0])
		self.visit(node.right[1])

	def NoValue(self, node):
		self.write('NoValue ')
		self.visit(node.value)

	def DeOpt(self, node):
		self.write('DeOpt ')
		self.visit(node.value)

	def For(self, node):
		self.visit(node.lvar)
		self.write(' <- ')
		self.visit(node.source)

	def LPad(self, node):
		self.write('LPad: {')
		for k, v in node.map.iteritems():
			self.visit(k)
			self.write(': ' + str(v))
		self.write('}')

	def LoopSetup(self, node):
		self.write('LoopSetup ')
		self.visit(node.loop)

	def LoopHeader(self, node):
		self.write('LoopHeader ctx:')
		self.visit(node.ctx)
		self.write(' lvar:')
		self.visit(node.lvar)
		self.write(' %s:%s' % (node.tg1, node.tg2))

	def Free(self, node):
		self.write('Free(')
		self.visit(node.value)
		self.write(')')

def prettify(name, flow):
	pp = PrettyPrinter()
	return pp.build(name, flow)
<commit_msg>Add a top-level comment to the pretty module<commit_after>'''A pretty printer for Runa CFG nodes.

This module implements another CFG walker, which spits out a string
represenation of the given CFG object. This is intended as a debugging aid
during compiler development, and can be invoked using the compiler driver's
`show` command (parametrized with the last pass to run before printing.
'''

from . import types

class PrettyPrinter(object):

	def __init__(self):
		self.buf = []

	def newline(self):
		self.buf.append('\n')

	def write(self, data):
		assert isinstance(data, str), data
		self.buf.append(data)

	def writeline(self, ln):
		if not ln: return
		self.write(ln + '\n')

	def visit(self, node):
		if isinstance(node, types.base):
			self.type(node)
		else:
			getattr(self, node.__class__.__name__)(node)

	def build(self, name, fun):

		name = '.'.join(name) if isinstance(name, tuple) else name
		self.start = False
		self.write('def %s(' % name)
		for i, node in enumerate(fun.args):
			self.visit(node)
			if i < len(fun.args) - 1:
				self.write(', ')

		self.write(')')
		if fun.rtype is not None:
			self.write(' -> ')
			self.visit(fun.rtype)

		self.write(':')
		self.newline()

		for i, bl in fun.flow.blocks.iteritems():
			self.writeline('  %2i: [%s]' % (i, bl.anno))
			for sid, step in enumerate(bl.steps):
				self.write(' {%02i} ' % sid)
				self.visit(step)
				self.newline()

		return ''.join(self.buf)

	def type(self, node):
		self.write(node.name)

	def anno(self, node):

		if node.type is None and not getattr(node, 'escapes', False):
			return

		if isinstance(node.type, (types.function, types.Type)):
			return

		self.write(' [')
		if node.type is not None:
			self.visit(node.type)

		if node.type is not None and getattr(node, 'escapes', False):
			self.write(':')

		if getattr(node, 'escapes', False):
			self.write('E')

		self.write(']')

	def NoneVal(self, node):
		self.write('NoneVal')
		self.anno(node)

	def Name(self, node):
		self.write(node.name)
		self.anno(node)

	def Bool(self, node):
		self.write(str(node.val))
		self.anno(node)

	def Int(self, node):
		self.write(str(node.val))
		self.anno(node)

	def Float(self, node):
		self.write(str(node.val))
		self.anno(node)

	def String(self, node):
		self.write(repr(node.val))
		self.anno(node)

	def binary(self, op, node):
		self.write(op + ' ')
		self.visit(node.left)
		self.write(' ')
		self.visit(node.right)

	def As(self, node):
		self.binary('As', node)

	def Not(self, node):
		self.write('Not ')
		self.visit(node.value)

	def And(self, node):
		self.binary('And', node)

	def Or(self, node):
		self.binary('Or', node)

	def Is(self, node):
		self.binary('Is', node)

	def NE(self, node):
		self.binary('NE', node)

	def GT(self, node):
		self.binary('GT', node)

	def LT(self, node):
		self.binary('LT', node)

	def EQ(self, node):
		self.binary('EQ', node)

	def Add(self, node):
		self.binary('Add', node)

	def Sub(self, node):
		self.binary('Sub', node)

	def Mod(self, node):
		self.binary('Mod', node)

	def Mul(self, node):
		self.binary('Mul', node)

	def Div(self, node):
		self.binary('Div', node)

	def BWAnd(self, node):
		self.binary('BWAnd', node)

	def BWOr(self, node):
		self.binary('BWOr', node)

	def BWXor(self, node):
		self.binary('BWXor', node)

	def Pass(self, node):
		self.write('Pass')

	def Init(self, node):
		self.write('Init ')
		self.visit(node.type)

	def Ref(self, node):
		self.write('&')
		self.visit(node.value)

	def Owner(self, node):
		self.write('$')
		self.visit(node.value)

	def Opt(self, node):
		self.visit(node.value)
		self.write('?')

	def Argument(self, node):
		self.write(node.name.name)
		self.anno(node)

	def Assign(self, node):
		self.visit(node.left)
		self.write(' = ')
		self.visit(node.right)

	def Call(self, node):
		self.visit(node.name)
		self.write('(')
		for i, arg in enumerate(node.args):
			self.visit(arg)
			if i < len(node.args) - 1:
				self.write(', ')
		self.write(')')
		self.anno(node)
		if node.callbr is not None:
			self.write(' => %s, %s' % node.callbr)

	def Return(self, node):
		self.write('Return')
		if node.value is not None:
			self.write(' ')
			self.visit(node.value)

	def Yield(self, node):
		self.write('Yield')
		if node.value is not None:
			self.write(' ')
			self.visit(node.value)

	def Raise(self, node):
		self.write('Raise ')
		self.visit(node.value)

	def CondBranch(self, node):
		self.write('CondBranch ')
		self.visit(node.cond)
		self.write(' ? %s : %s' % (node.tg1, node.tg2))

	def Branch(self, node):
		self.write('Branch %s' % node.label)

	def Attrib(self, node):
		self.visit(node.obj)
		self.write(' . ')
		self.write(node.attrib)
		self.anno(node)

	def SetAttr(self, node):
		self.Attrib(node)

	def Elem(self, node):
		self.write('Elem(')
		self.visit(node.obj)
		self.write(', ')
		self.visit(node.key)
		self.write(')')
		self.anno(node)

	def Phi(self, node):
		self.write('Phi ')
		self.write('%s:' % node.left[0])
		self.visit(node.left[1])
		self.write(', ')
		self.write('%i:' % node.right[0])
		self.visit(node.right[1])

	def NoValue(self, node):
		self.write('NoValue ')
		self.visit(node.value)

	def DeOpt(self, node):
		self.write('DeOpt ')
		self.visit(node.value)

	def For(self, node):
		self.visit(node.lvar)
		self.write(' <- ')
		self.visit(node.source)

	def LPad(self, node):
		self.write('LPad: {')
		for k, v in node.map.iteritems():
			self.visit(k)
			self.write(': ' + str(v))
		self.write('}')

	def LoopSetup(self, node):
		self.write('LoopSetup ')
		self.visit(node.loop)

	def LoopHeader(self, node):
		self.write('LoopHeader ctx:')
		self.visit(node.ctx)
		self.write(' lvar:')
		self.visit(node.lvar)
		self.write(' %s:%s' % (node.tg1, node.tg2))

	def Free(self, node):
		self.write('Free(')
		self.visit(node.value)
		self.write(')')

def prettify(name, flow):
	pp = PrettyPrinter()
	return pp.build(name, flow)
<|endoftext|>


<commit_before>#!/usr/bin/python

import distutils.dir_util
import os
import shutil
import subprocess
import sys
import zipfile

SRC_DIR = 'src'
DIST_DIR = 'dist'
ZIP_DIR = 'zip'
ELI_DIR = '../eli'
EL_DIR = '../el'
DEVENV_DIR = DIST_DIR+'/devenv'

PATCHES_SOURCE = 'E:/versions/plw6/release/610SP1/updates/_en_dev/' if sys.platform == 'win32' else '/E/versions/plw6/release/610SP1/updates/_en_dev/'

TARGETS = {'zip' : ['clean', 'init', 'copyfiles', 'version', 'patches', 'buildzip'],
           'env' : ['init', 'copyfiles'],
           'clean' : ['clean']}

def make(target):
    for _function in TARGETS[target]:
        getattr(sys.modules[__name__],_function)()

def getversion():
    if os.getenv('VERSION'):
        return os.getenv('VERSION')
    elif os.getenv('BUILD_NUMBER'):
        return "incremental-"+os.getenv('BUILD_NUMBER')
    else:
        tag = subprocess.check_output(['git', 'describe', '--tags', '--abbrev=10']).strip("\n\r ")
        return "incremental-"+tag

def buildzip():
    filename = 'emacs-runtime-'+getversion()+".zip"
    print "Building "+filename
    base = DIST_DIR + '/'
    with zipfile.ZipFile(ZIP_DIR+ '/'+filename, 'w') as _zipfile:
        ## zip the content of the dist directory as base
        for subdir, dirs, files in os.walk(DIST_DIR):
            for _f in files:
                _file = os.path.join(subdir, _f)
                _zipfile.write(_file, _file[len(base):] if _file.startswith(base) else _file)

def patches():
    destdir = DIST_DIR + '/patches'
    if not os.path.isdir(destdir):
        os.mkdir(destdir)
    with open('patches', 'r') as _f:
        for patch in _f.readlines():
            shutil.copy2(PATCHES_SOURCE + patch.strip("\n\r ") + '-is.obin', destdir)

def version():
    vfile = DIST_DIR + '/version.txt'
    if os.path.isfile(vfile):
        os.remove(vfile)
    with open(vfile, 'w') as _f:
        _f.write(getversion() + '\n')
        tag = subprocess.check_output(['git', 'describe', '--tags', '--abbrev=100']).strip("\n\r ")
        _f.write(tag + '\n')

def init():
    for _dir in [DIST_DIR, ZIP_DIR, DEVENV_DIR, DEVENV_DIR+'/el']:
        if not os.path.isdir(_dir):
            os.mkdir(_dir)

def copyfiles():
    print "Copying files"

    print "Copying source files"
    copyall(SRC_DIR, DIST_DIR)

    print "Copying eli files"
    distutils.dir_util.copy_tree(ELI_DIR, DIST_DIR+'/eli')

    print "Copying el files"
    copyall(EL_DIR, DEVENV_DIR + '/el')
    shutil.copy('../emacs-plw-ext.el', DEVENV_DIR)

def clean():
    for _dir in [DIST_DIR,ZIP_DIR]:
        if os.path.isdir(_dir):
            shutil.rmtree(_dir)

def copyall(src, dst, symlinks=False, ignore=None):
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            if not os.path.isdir(d):
                os.mkdir(d)

            copyall(s, d, symlinks, ignore)
        else:
            shutil.copy2(s, d)

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print "Usage : ./make.py target"
        print "  target can be : " + ", ".join(TARGETS.keys())
        sys.exit(1)
    else:
        make(sys.argv[1])
<commit_msg>* python3<commit_after>#!/usr/bin/python3
## -*- coding: windows-1252 -*-
##  COPYRIGHT (C) PLANISWARE 2018
##
##  All Rights Reserved
##
##  This program and the information contained herein are confidential to
##  and the property of PLANISWARE and are made available only to PLANISWARE
##  employees for the sole purpose of conducting PLANISWARE business.
##
##**************************************************************************

import distutils.dir_util
import os
import shutil
import subprocess
import sys
import zipfile

SRC_DIR = 'src'
DIST_DIR = 'dist'
ZIP_DIR = 'zip'
ELI_DIR = '../eli'
EL_DIR = '../el'
DEVENV_DIR = DIST_DIR+'/devenv'

PATCHES_SOURCE = 'E:/versions/plw6/release/610SP1/updates/_en_dev/' if sys.platform == 'win32' else '/E/versions/plw6/release/610SP1/updates/_en_dev/'

TARGETS = {'zip' : ['clean', 'init', 'copyfiles', 'version', 'patches', 'buildzip'],
       'env' : ['init', 'copyfiles'],
       'clean' : ['clean']}

def make(target):
  if target not in TARGETS:
    usage()

  for _function in TARGETS[target]:
    getattr(sys.modules[__name__],_function)()

def getversion():
  if os.getenv('VERSION'):
    return os.getenv('VERSION')
  elif os.getenv('BUILD_NUMBER'):
    return "incremental-"+os.getenv('BUILD_NUMBER')
  else:
    tag = subprocess.check_output(['git', 'describe', '--tags', '--abbrev=10']).strip("\n\r ")
    return "incremental-"+tag

def buildzip():
  filename = 'emacs-runtime-'+getversion()+".zip"
  print("Building "+filename)
  base = DIST_DIR + '/'
  with zipfile.ZipFile(ZIP_DIR+ '/'+filename, 'w') as _zipfile:
    ## zip the content of the dist directory as base
    for subdir, dirs, files in os.walk(DIST_DIR):
      for _f in files:
        _file = os.path.join(subdir, _f)
        _zipfile.write(_file, _file[len(base):] if _file.startswith(base) else _file)

def patches():
  destdir = DIST_DIR + '/patches'
  if not os.path.isdir(destdir):
    os.mkdir(destdir)
  with open('patches', 'r') as _f:
    for patch in _f.readlines():
      shutil.copy2(PATCHES_SOURCE + patch.strip("\n\r ") + '-is.obin', destdir)

def version():
  vfile = DIST_DIR + '/version.txt'
  if os.path.isfile(vfile):
    os.remove(vfile)
  with open(vfile, 'w') as _f:
    _f.write(getversion() + '\n')
    tag = subprocess.check_output(['git', 'describe', '--tags', '--abbrev=100']).strip("\n\r ")
    _f.write(tag + '\n')

def init():
  for _dir in [DIST_DIR, ZIP_DIR, DEVENV_DIR, DEVENV_DIR+'/el']:
    if not os.path.isdir(_dir):
      os.mkdir(_dir)

def copyfiles():
  print("Copying files")

  print("Copying source files")
  copyall(SRC_DIR, DIST_DIR)

  print("Copying eli files")
  distutils.dir_util.copy_tree(ELI_DIR, DIST_DIR+'/eli')

  print("Copying el files")
  copyall(EL_DIR, DEVENV_DIR + '/el')
  shutil.copy('../emacs-plw-ext.el', DEVENV_DIR)

def clean():
  for _dir in [DIST_DIR,ZIP_DIR]:
    if os.path.isdir(_dir):
      shutil.rmtree(_dir)

def copyall(src, dst, symlinks=False, ignore=None):
  for item in os.listdir(src):
    s = os.path.join(src, item)
    d = os.path.join(dst, item)
    if os.path.isdir(s):
      if not os.path.isdir(d):
        os.mkdir(d)

      copyall(s, d, symlinks, ignore)
    else:
      shutil.copy2(s, d)

def usage():
  print("Usage : ./make.py target")
  print("  target can be : " + ", ".join(TARGETS.keys()))
  sys.exit(1)

if __name__ == '__main__':
  if len(sys.argv) != 2:
    usage()
  else:
    make(sys.argv[1])
<|endoftext|>


<commit_before>#!python
from __future__ import print_function
import os
import sys
import socket
import time
import ctypes as c
import json

if sys.platform != "win32":
	# simple sanity check
	raise RuntimeError("This script requires Windows NT with Terminal Services.")

import servicemanager as sm
import win32api as Api
from win32con import *
from win32gui import *
from win32ts import *
from win32security import *
import win32service
import win32serviceutil

SERVER_URL = "http://equal.cluenet.org/rwho/server.php"

## Windows native types and constants

# generic types
UINT				= c.c_uint
WORD				= c.c_uint16
DWORD				= c.c_uint32
WCHAR				= c.c_wchar
HANDLE				= DWORD
HWND				= HANDLE

# window messages
WM_WTSSESSION_CHANGE		= 0x2B1

# WM_WTSSESSION_CHANGE events
WTS_CONSOLE_CONNECT		= 0x1
WTS_CONSOLE_DISCONNECT		= 0x2
WTS_REMOTE_CONNECT		= 0x3
WTS_REMOTE_DISCONNECT		= 0x4
WTS_SESSION_LOGON		= 0x5
WTS_SESSION_LOGOFF		= 0x6
WTS_SESSION_LOCK		= 0x7
WTS_SESSION_UNLOCK		= 0x8
WTS_SESSION_REMOTE_CONTROL	= 0x9

# http://msdn.microsoft.com/en-us/library/aa383860(v=vs.85).aspx
WTS_CONNECTSTATE_CLASS	= DWORD # enum
WTSActive		= 0
WTSConnected		= 1
WTSConnectQuery		= 2
WTSShadow		= 3
WTSDisconnected		= 4
WTSIdle			= 5
WTSListen		= 6
WTSReset		= 7
WTSDown			= 8
WTSInit			= 9

# http://msdn.microsoft.com/en-us/library/aa383861(v=vs.85).aspx
WTS_INFO_CLASS		= DWORD # enum
WTSSessionInfo		= 24

# include <winsta.h>
# http://msdn.microsoft.com/en-us/library/cc248871(PROT.10).aspx
WINSTATIONNAME_LENGTH		= 32
USERNAME_LENGTH			= 20
DOMAIN_LENGTH			= 17

class LARGE_INTEGER(c.Structure):
	_fields_ = (
		("HighPart",	DWORD),
		("LowPart",	DWORD),
	)

class FILETIME(c.Structure):
	_fields_ = (
		("dwHighDateTime",	DWORD),
		("dwLowDateTime",	DWORD),
	)

class WTSINFO(c.Structure):
	# http://msdn.microsoft.com/en-us/library/bb736370(v=vs.85).aspx
	_fields_ = (
		("State",			WTS_CONNECTSTATE_CLASS),
		("SessionId",			DWORD),
		("IncomingBytes",		DWORD),
		("OutgoingBytes",		DWORD),
		("IncomingFrames",		DWORD),
		("OutgoingFrames",		DWORD),
		("IncomingCompressedBytes",	DWORD),
		("OutgoingCompressedBytes",	DWORD),
		("WinStationName",		WCHAR * (WINSTATIONNAME_LENGTH+1)),
		("Domain",			WCHAR * (DOMAIN_LENGTH+1)),
		("UserName",			WCHAR * (USERNAME_LENGTH+1)),
		("ConnectTime",			FILETIME),
		("DisconnectTime",		FILETIME),
		("LastInputTime",		FILETIME),
		("LogonTime",			FILETIME),
		("CurrentTime",			FILETIME),
	)

## Windows native functions

def _wtsapi_WTSQuerySessionInformation(hServer, sessionID, infoClass):
	ppBuffer = DWORD(0)
	pBytesReturned = DWORD(0)

	res = c.windll.wtsapi32.WTSQuerySessionInformationW(
		HANDLE(hServer), DWORD(sessionID), WTS_INFO_CLASS(infoClass),
		c.byref(ppBuffer), c.byref(pBytesReturned))

	if res:
		return (ppBuffer, pBytesReturned)
	else:
		print("WTSQuerySessionInformationW: error %r" % Api.GetLastError())
		return (0, 0)

def WTSQuerySessionInformation_SessionInfo(hServer, sessionId):
	# only supported starting with Server 2008 & Vista SP1
	buf, bufsize = _wtsapi_WTSQuerySessionInformation(hServer, sessionId, WTSSessionInfo)
	if bufsize:
		return c.cast(buf.value, c.POINTER(WTSINFO)).contents
	else:
		return None

def SetTimer(hWnd, IDEvent, elapse):
	IDEvent = UINT(IDEvent)
	if c.windll.user32.SetTimer(HWND(hWnd), c.byref(IDEvent), UINT(elapse), None):
		return IDEvent.value
	else:
		return False

def UnixTimeFromFileTime(ftime):
	return (((ftime.dwHighDateTime << 32) | ftime.dwLowDateTime) - 116444736000000000) / 10000000.0

def collect_session_info():
	hServer = WTS_CURRENT_SERVER_HANDLE
	#hServer = WTSOpenServer("...")
	for sess in WTSEnumerateSessions(hServer):
		print("session", sess)

		if sess["State"] != WTSActive:
			# skip inactive (disconnected) sessions
			continue

		sessionId = sess["SessionId"]
		user = WTSQuerySessionInformation(hServer, sessionId, WTSUserName)
		if not user:
			print("skipping (null user)")
			continue

		entry = {}
		entry["user"] = user
		entry["line"] = sess["WinStationName"]
		entry["host"] = WTSQuerySessionInformation(hServer, sessionId, WTSClientName)

		sessionInfo = WTSQuerySessionInformation_SessionInfo(hServer, sessionId)
		if sessionInfo:
			print("logonTime", sessionInfo.LogonTime)
			entry["time"] = int(UnixTimeFromFileTime(sessionInfo.LogonTime))
		else:
			entry["time"] = 0

		if hServer == WTS_CURRENT_SERVER_HANDLE:
			uSid, uDom, acctType = LookupAccountName(None, user)
			uSidAuthorities = [uSid.GetSubAuthority(i)
						for i in range(uSid.GetSubAuthorityCount())]
			entry["uid"] = uSidAuthorities[-1]
			entry["domain"] = uDom
		else:
			entry["uid"] = 0
			entry["domain"] = None

		yield entry

def update():
	upload("put", list(collect_session_info()))

def cleanup():
	upload("destroy", [])

def upload(action, sdata):
	try:
		from urllib import urlencode
		from urllib2 import urlopen
	except ImportError:
		from urllib.parse import urlencode
		from urllib.request import urlopen

	print("uploading %d items" % len(sdata))
	data = {
		"host": socket.gethostname().lower(),
		"fqdn": socket.getfqdn().lower(),
		"action": action,
		"utmp": json.dumps(sdata),
	}
	data = urlencode(data).encode("utf-8")
	resp = urlopen(SERVER_URL, data)
	print("server returned: %r" % resp.readline().strip())

class WTSSessionEventMonitor():
	"""Windows Terminal Services session event monitor"""

	className = "Monitor"
	wndName = "Monitor"

	def __init__(self):
		wc = WNDCLASS()
		wc.hInstance     = GetModuleHandle(None)
		wc.lpszClassName = self.className
		wc.lpfnWndProc   = self.WndProc

		self.classAtom = RegisterClass(wc)

		self.hWnd = CreateWindow(self.classAtom, self.wndName,
			# style, x, y, nWidth, nHeight,
			0, 0, 0, CW_USEDEFAULT, CW_USEDEFAULT,
			# hWndParent, hMenu, hInstance, lpParam
			0, 0, wc.hInstance, None)
		UpdateWindow(self.hWnd)

		WTSRegisterSessionNotification(self.hWnd, NOTIFY_FOR_ALL_SESSIONS)

	def WndProc(self, hWnd, message, wParam, lParam):
		if message == WM_CLOSE:
			DestroyWindow(hWnd)
		elif message == WM_DESTROY:
			PostQuitMessage(0)
		elif message == WM_QUERYENDSESSION:
			return True
		else:
			return DefWindowProc(hWnd, message, wParam, lParam)

class RWhoMonitor(WTSSessionEventMonitor):
	def __init__(self):
		WTSSessionEventMonitor.__init__(self)

		self.running = False
		self.timerId = 42
		self.periodic_timeout = 10*60

		try:
			Api.GetConsoleTitle()
		except:
			self.have_console = False
		else:
			self.have_console = True

	def run(self):
		print("starting monitor")
		try:
			if self.have_console:
				Api.SetConsoleCtrlHandler(self.OnConsoleCtrlEvent, True)

			self.running = True
			self.OnTimer()
			PumpMessages()
		except:
			self.running = False
			raise

	def stop(self):
		print("stopping monitor")
		try:
			self.running = False

			cleanup()
			SendMessage(self.hWnd, WM_DESTROY, 0, 0)
		except:
			self.running = True
			raise

	def WndProc(self, hWnd, message, wParam, lParam):
		DefWndProc = lambda: \
			WTSSessionEventMonitor.WndProc(self, hWnd, message, wParam, lParam)

		if message == WM_POWERBROADCAST:
			if wParam == PBT_APMSUSPEND:
				self.OnSuspend()
			elif wParam == PBT_APMRESUMESUSPEND:
				self.OnResume()
			return DefWndProc()
		elif message == WM_TIMER:
			self.OnTimer()
		elif message == WM_WTSSESSION_CHANGE:
			self.OnSession(wParam, lParam)
		elif message == WM_ENDSESSION:
			self.OnShutdown()
			return True
		elif message == WM_DESTROY:
			self.OnDestroy()
			return DefWndProc()
		else:
			return DefWndProc()

	def OnConsoleCtrlEvent(self, event):
		# http://msdn.microsoft.com/en-us/library/ms683242%28v=vs.85%29.aspx
		print("* console CtrlEvent(%d)" % event)
		if not self.running:
			return

		self.stop()

	def OnSuspend(self):
		print("* suspend")
		if not self.running:
			return

		cleanup()

	def OnResume(self):
		print("* resume")
		if not self.running:
			return

		update()

	def OnShutdown(self):
		print("* shutdown")
		if not self.running:
			return

		cleanup()

	def OnDestroy(self):
		print("* destroy")
		if not self.running:
			return

		cleanup()

	def OnTimer(self):
		print("* timer")
		if not self.running:
			return

		self.timerId = SetTimer(self.hWnd, self.timerId,
					self.periodic_timeout*1000)
		update()

	def OnSession(self, event, session):
		if not self.running:
			return

		event_name = {
			WTS_CONSOLE_CONNECT:		"connected to console",
			WTS_CONSOLE_DISCONNECT:		"disconnected from console",
			WTS_REMOTE_CONNECT:		"connected remotely",
			WTS_REMOTE_DISCONNECT:		"disconnected remotely",
			WTS_SESSION_LOGON:		"logged on",
			WTS_SESSION_LOGOFF:		"logged off",
			WTS_SESSION_LOCK:		"locked",
			WTS_SESSION_UNLOCK:		"unlocked",
			WTS_SESSION_REMOTE_CONTROL:	"remote control",
		}.get(event, "unknown %d" % event)

		print("* session %d %s" % (session, event_name))
		update()

class RWhoService(win32serviceutil.ServiceFramework):
	_svc_name_ = "rwhod"
	_svc_display_name_ = "rwho daemon"

	def __init__(self, args):
		win32serviceutil.ServiceFramework.__init__(self, args)

	def SvcStop(self):
		self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
		cleanup()
		PostQuitMessage(0)
		self.ReportServiceStatus(win32service.SERVICE_STOPPED)

	def SvcDoRun(self):
		m = RWhoMonitor()

if __name__ == '__main__':
	try:
		if len(sys.argv) > 1:
			win32serviceutil.HandleCommandLine(RWhoService)
		else:
			mon = RWhoMonitor()
			try:
				mon.run()
			except KeyboardInterrupt:
				# keyboard interrupts will be handled by monitor itself
				pass
	except BaseException as e:
		path = os.path.expandvars("%TEMP%/rwho.log")
		with open(path, "a") as log:
			print(repr(e), file=log)
		raise
<commit_msg>winnt: memleak? *shrug*<commit_after>#!python
from __future__ import print_function
import os
import sys
import socket
import time
import ctypes as c
import json

if sys.platform != "win32":
	# simple sanity check
	raise RuntimeError("This script requires Windows NT with Terminal Services.")

import servicemanager as sm
import win32api as Api
from win32con import *
from win32gui import *
from win32ts import *
from win32security import *
import win32service
import win32serviceutil

SERVER_URL = "http://equal.cluenet.org/rwho/server.php"

## Windows native types and constants

# generic types
UINT				= c.c_uint
WORD				= c.c_uint16
DWORD				= c.c_uint32
WCHAR				= c.c_wchar
HANDLE				= DWORD
HWND				= HANDLE

# window messages
WM_WTSSESSION_CHANGE		= 0x2B1

# WM_WTSSESSION_CHANGE events
WTS_CONSOLE_CONNECT		= 0x1
WTS_CONSOLE_DISCONNECT		= 0x2
WTS_REMOTE_CONNECT		= 0x3
WTS_REMOTE_DISCONNECT		= 0x4
WTS_SESSION_LOGON		= 0x5
WTS_SESSION_LOGOFF		= 0x6
WTS_SESSION_LOCK		= 0x7
WTS_SESSION_UNLOCK		= 0x8
WTS_SESSION_REMOTE_CONTROL	= 0x9

# http://msdn.microsoft.com/en-us/library/aa383860(v=vs.85).aspx
WTS_CONNECTSTATE_CLASS	= DWORD # enum
WTSActive		= 0
WTSConnected		= 1
WTSConnectQuery		= 2
WTSShadow		= 3
WTSDisconnected		= 4
WTSIdle			= 5
WTSListen		= 6
WTSReset		= 7
WTSDown			= 8
WTSInit			= 9

# http://msdn.microsoft.com/en-us/library/aa383861(v=vs.85).aspx
WTS_INFO_CLASS		= DWORD # enum
WTSSessionInfo		= 24

# include <winsta.h>
# http://msdn.microsoft.com/en-us/library/cc248871(PROT.10).aspx
WINSTATIONNAME_LENGTH		= 32
USERNAME_LENGTH			= 20
DOMAIN_LENGTH			= 17

class LARGE_INTEGER(c.Structure):
	_fields_ = (
		("HighPart",	DWORD),
		("LowPart",	DWORD),
	)

class FILETIME(c.Structure):
	_fields_ = (
		("dwHighDateTime",	DWORD),
		("dwLowDateTime",	DWORD),
	)

class WTSINFO(c.Structure):
	# http://msdn.microsoft.com/en-us/library/bb736370(v=vs.85).aspx
	_fields_ = (
		("State",			WTS_CONNECTSTATE_CLASS),
		("SessionId",			DWORD),
		("IncomingBytes",		DWORD),
		("OutgoingBytes",		DWORD),
		("IncomingFrames",		DWORD),
		("OutgoingFrames",		DWORD),
		("IncomingCompressedBytes",	DWORD),
		("OutgoingCompressedBytes",	DWORD),
		("WinStationName",		WCHAR * (WINSTATIONNAME_LENGTH+1)),
		("Domain",			WCHAR * (DOMAIN_LENGTH+1)),
		("UserName",			WCHAR * (USERNAME_LENGTH+1)),
		("ConnectTime",			FILETIME),
		("DisconnectTime",		FILETIME),
		("LastInputTime",		FILETIME),
		("LogonTime",			FILETIME),
		("CurrentTime",			FILETIME),
	)

## Windows native functions

def _wtsapi_WTSQuerySessionInformation(hServer, sessionID, infoClass):
	ppBuffer = DWORD(0)
	pBytesReturned = DWORD(0)

	res = c.windll.wtsapi32.WTSQuerySessionInformationW(
		HANDLE(hServer), DWORD(sessionID), WTS_INFO_CLASS(infoClass),
		c.byref(ppBuffer), c.byref(pBytesReturned))

	if res:
		return (ppBuffer, pBytesReturned)
	else:
		print("WTSQuerySessionInformationW: error %r" % Api.GetLastError())
		return (0, 0)

def WTSQuerySessionInformation_SessionInfo(hServer, sessionId):
	# only supported starting with Server 2008 & Vista SP1
	buf, bufsize = _wtsapi_WTSQuerySessionInformation(hServer, sessionId, WTSSessionInfo)
	if bufsize:
		return buf, c.cast(buf.value, c.POINTER(WTSINFO)).contents
	else:
		return None

def SetTimer(hWnd, IDEvent, elapse):
	IDEvent = UINT(IDEvent)
	if c.windll.user32.SetTimer(HWND(hWnd), c.byref(IDEvent), UINT(elapse), None):
		return IDEvent.value
	else:
		return False

def UnixTimeFromFileTime(ftime):
	return (((ftime.dwHighDateTime << 32) | ftime.dwLowDateTime) - 116444736000000000) / 10000000.0

def collect_session_info():
	hServer = WTS_CURRENT_SERVER_HANDLE
	#hServer = WTSOpenServer("...")
	for sess in WTSEnumerateSessions(hServer):
		print("session", sess)

		if sess["State"] != WTSActive:
			# skip inactive (disconnected) sessions
			continue

		sessionId = sess["SessionId"]
		user = WTSQuerySessionInformation(hServer, sessionId, WTSUserName)
		if not user:
			print("skipping (null user)")
			continue

		entry = {}
		entry["user"] = user
		entry["line"] = sess["WinStationName"]
		entry["host"] = WTSQuerySessionInformation(hServer, sessionId, WTSClientName)

		ptr, sessionInfo = WTSQuerySessionInformation_SessionInfo(hServer, sessionId)
		# only supported starting with Server 2008 & Vista SP1
		if sessionInfo:
			print("logonTime", sessionInfo.LogonTime)
			entry["time"] = int(UnixTimeFromFileTime(sessionInfo.LogonTime))
			c.windll.wtsapi32.WTSFreeMemory(ptr)
		else:
			entry["time"] = 0

		if hServer == WTS_CURRENT_SERVER_HANDLE:
			uSid, uDom, acctType = LookupAccountName(None, user)
			uSidAuthorities = [uSid.GetSubAuthority(i)
						for i in range(uSid.GetSubAuthorityCount())]
			entry["uid"] = uSidAuthorities[-1]
			entry["domain"] = uDom
		else:
			entry["uid"] = 0
			entry["domain"] = None

		yield entry

def update():
	upload("put", list(collect_session_info()))

def cleanup():
	upload("destroy", [])

def upload(action, sdata):
	try:
		from urllib import urlencode
		from urllib2 import urlopen
	except ImportError:
		from urllib.parse import urlencode
		from urllib.request import urlopen

	print("uploading %d items" % len(sdata))
	data = {
		"host": socket.gethostname().lower(),
		"fqdn": socket.getfqdn().lower(),
		"action": action,
		"utmp": json.dumps(sdata),
	}
	data = urlencode(data).encode("utf-8")
	resp = urlopen(SERVER_URL, data)
	print("server returned: %r" % resp.readline().strip())

class WTSSessionEventMonitor():
	"""Windows Terminal Services session event monitor"""

	className = "Monitor"
	wndName = "Monitor"

	def __init__(self):
		wc = WNDCLASS()
		wc.hInstance     = GetModuleHandle(None)
		wc.lpszClassName = self.className
		wc.lpfnWndProc   = self.WndProc

		self.classAtom = RegisterClass(wc)

		self.hWnd = CreateWindow(self.classAtom, self.wndName,
			# style, x, y, nWidth, nHeight,
			0, 0, 0, CW_USEDEFAULT, CW_USEDEFAULT,
			# hWndParent, hMenu, hInstance, lpParam
			0, 0, wc.hInstance, None)
		UpdateWindow(self.hWnd)

		WTSRegisterSessionNotification(self.hWnd, NOTIFY_FOR_ALL_SESSIONS)

	def WndProc(self, hWnd, message, wParam, lParam):
		if message == WM_CLOSE:
			DestroyWindow(hWnd)
		elif message == WM_DESTROY:
			PostQuitMessage(0)
		elif message == WM_QUERYENDSESSION:
			return True
		else:
			return DefWindowProc(hWnd, message, wParam, lParam)

class RWhoMonitor(WTSSessionEventMonitor):
	def __init__(self):
		WTSSessionEventMonitor.__init__(self)

		self.running = False
		self.timerId = 42
		self.periodic_timeout = 10*60

		try:
			Api.GetConsoleTitle()
		except:
			self.have_console = False
		else:
			self.have_console = True

	def run(self):
		print("starting monitor")
		try:
			if self.have_console:
				Api.SetConsoleCtrlHandler(self.OnConsoleCtrlEvent, True)

			self.running = True
			self.OnTimer()
			PumpMessages()
		except:
			self.running = False
			raise

	def stop(self):
		print("stopping monitor")
		try:
			self.running = False

			cleanup()
			SendMessage(self.hWnd, WM_DESTROY, 0, 0)
		except:
			self.running = True
			raise

	def WndProc(self, hWnd, message, wParam, lParam):
		DefWndProc = lambda: \
			WTSSessionEventMonitor.WndProc(self, hWnd, message, wParam, lParam)

		if message == WM_POWERBROADCAST:
			if wParam == PBT_APMSUSPEND:
				self.OnSuspend()
			elif wParam == PBT_APMRESUMESUSPEND:
				self.OnResume()
			return DefWndProc()
		elif message == WM_TIMER:
			self.OnTimer()
		elif message == WM_WTSSESSION_CHANGE:
			self.OnSession(wParam, lParam)
		elif message == WM_ENDSESSION:
			self.OnShutdown()
			return True
		elif message == WM_DESTROY:
			self.OnDestroy()
			return DefWndProc()
		else:
			return DefWndProc()

	def OnConsoleCtrlEvent(self, event):
		# http://msdn.microsoft.com/en-us/library/ms683242%28v=vs.85%29.aspx
		print("* console CtrlEvent(%d)" % event)
		if not self.running:
			return

		self.stop()

	def OnSuspend(self):
		print("* suspend")
		if not self.running:
			return

		cleanup()

	def OnResume(self):
		print("* resume")
		if not self.running:
			return

		update()

	def OnShutdown(self):
		print("* shutdown")
		if not self.running:
			return

		cleanup()

	def OnDestroy(self):
		print("* destroy")
		if not self.running:
			return

		cleanup()

	def OnTimer(self):
		print("* timer")
		if not self.running:
			return

		self.timerId = SetTimer(self.hWnd, self.timerId,
					self.periodic_timeout*1000)
		update()

	def OnSession(self, event, session):
		if not self.running:
			return

		event_name = {
			WTS_CONSOLE_CONNECT:		"connected to console",
			WTS_CONSOLE_DISCONNECT:		"disconnected from console",
			WTS_REMOTE_CONNECT:		"connected remotely",
			WTS_REMOTE_DISCONNECT:		"disconnected remotely",
			WTS_SESSION_LOGON:		"logged on",
			WTS_SESSION_LOGOFF:		"logged off",
			WTS_SESSION_LOCK:		"locked",
			WTS_SESSION_UNLOCK:		"unlocked",
			WTS_SESSION_REMOTE_CONTROL:	"remote control",
		}.get(event, "unknown %d" % event)

		print("* session %d %s" % (session, event_name))
		update()

class RWhoService(win32serviceutil.ServiceFramework):
	_svc_name_ = "rwhod"
	_svc_display_name_ = "rwho daemon"

	def __init__(self, args):
		win32serviceutil.ServiceFramework.__init__(self, args)

	def SvcStop(self):
		self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
		cleanup()
		PostQuitMessage(0)
		self.ReportServiceStatus(win32service.SERVICE_STOPPED)

	def SvcDoRun(self):
		m = RWhoMonitor()

if __name__ == '__main__':
	try:
		if len(sys.argv) > 1:
			win32serviceutil.HandleCommandLine(RWhoService)
		else:
			mon = RWhoMonitor()
			try:
				mon.run()
			except KeyboardInterrupt:
				# keyboard interrupts will be handled by monitor itself
				pass
	except BaseException as e:
		path = os.path.expandvars("%TEMP%/rwho.log")
		with open(path, "a") as log:
			print(repr(e), file=log)
		raise
<|endoftext|>


<commit_before># Copyright (C) 2012 Nippon Telegraph and Telephone Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version_info = (3, 8)
version = '.'.join(map(str, version_info))
<commit_msg>Ryu 3.9<commit_after># Copyright (C) 2012 Nippon Telegraph and Telephone Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version_info = (3, 9)
version = '.'.join(map(str, version_info))
<|endoftext|>


<commit_before>import yaml
import itertools
import sys
import argh
import s
import os

_services = './docker/state/services.yml'

@argh.arg('cmd', nargs='?', default=None)
def run(action_name, cmd, tty=False, no_clean_before=False, no_clean_after=False):
    """run an action"""
    build(action_name)
    tty = True if cmd == 'bash' else tty
    action = _get_action(action_name)
    if not no_clean_before:
        stop(action_name)
    try:
        _start_deps(action)
        print('start main')
        try:
            s.shell.run(_run_cmd(action['main'], tty=tty, cmd=cmd), interactive=True)
        except:
            sys.exit(1)
    finally:
        if not no_clean_after:
            stop(action_name)

def ports(action_name, container_name=None):
    action = _get_action(action_name)
    if container_name:
        print(_exposed_port(action[container_name]['tag']))
    else:
        for container_name, data in action.items():
            port = _exposed_port(data['tag'])
            if port:
                print('{container_name}:{port}'.format(**locals()))

def stop(action_name):
    action = _get_action(action_name)
    for container_name, data in action.items():
        with s.exceptions.ignore():
            s.shell.run('sudo docker kill', data['tag'])
            s.shell.run('sudo docker rm', data['tag'])

def build(action_name, container=None, nocache=False, pull=False, force=False):
    """build an actions containers"""
    action = _get_action(action_name)
    for container_name, data in action.items():
        if not container or container_name == container:
            if force or not s.shell.run('sudo docker inspect', data['tag'], zero=True, stream=False):
                s.shell.run(_build_cmd(data, nocache, pull), stream=True)
                print(s.colors.green('built:'), data['tag'])

def show():
    """show available actions"""
    data = _get_actions()
    for action_name in sorted(data):
        print('')
        print(action_name)
        for container in data[action_name]:
            print('', container)

def main():
    assert os.path.isdir('docker'), 'must have a dir: ./docker'
    assert os.path.isfile('system.yml'), 'must have a file: ./system.yml'
    if s.shell.override('--stream'):
        with s.shell.set_stream():
            _main()
    else:
        _main()

def _main():
    s.shell.run('mkdir -p ./docker/state')
    s.shell.dispatch_commands(globals(), __name__)

def _exposed_port(tag):
    port = s.shell.run('sudo docker port', tag).split(':')[-1]
    if port:
        return int(port)

def _start_deps(action):
    s.shell.run('rm -f', _services)
    to_start = list(s.dicts.drop(action, 'main').items())
    if to_start:
        started = []
        for i in itertools.count():
            container_name, data = to_start.pop()
            deps = data.get('depends', [])
            if not deps or all(x in started for x in deps):
                print('start dep:', container_name)
                s.shell.run(_run_cmd(data, bg=True))
                started.append(container_name)
                data = {'port': _exposed_port(data['tag']),
                        'host': _host_ip()}
                with open(_services, 'a') as f:
                    f.write(yaml.dump({container_name: data}))
                s.web.wait_for_http('http://{host}:{port}'.format(**data))
            else:
                to_start.append([container_name, data])
            assert i < 500, 'never resolved dependency order. remaining: {}'.format(to_start)
            if not to_start:
                return

def _tag_containers(action_name, action):
    for container_name, data in action.items():
        action[container_name]['tag'] = '_'.join([
            os.path.basename(os.getcwd()),
            action_name,
            container_name
        ])
    return action

def _get_actions():
    with open('./system.yml') as f:
        return yaml.safe_load(f)

def _get_action(action_name):
    action = _get_actions()[action_name]
    return _tag_containers(action_name, action)

def _build_cmd(data, nocache, pull):
    return ' '.join([
        'sudo docker build',
        '-t', data['tag'],
        '--force-rm=true',
        '-f ' + data['dockerfile'] if 'dockerfile' in data else '',
        '--no-cache=true' if nocache else '--no-cache=false',
        '--pull=true' if pull else '--pull=false',
        '.',
    ])

def _run_cmd(data, tty=False, cmd=None, bg=False):
    return ' '.join(
        ['sudo docker run',
         '--publish-all=true',
         '-it' if tty else '',
         '-d' if bg else '',
         '--name={tag}'.format(**data),
         '-v {}/docker/state:/state'.format(os.getcwd())] +
        ['-v {}:{}'.format(os.path.abspath(os.path.expanduser(a)), b)
         for x in data.get('volumes', [])
         for y in [str(x).split(':')]
         for a, b in [(y + y)[:2]]] +
        ['-p {}:{}'.format(*z)
         for x in data.get('ports', [])
         for y in [str(x).split(':')]
         for z in [([s.net.free_port()] + y)[-2:]]] +
        [data['tag']] +
        ["bash -c '{}'".format(cmd)
         if cmd
         else "bash -c '{cmd}'".format(**data)
         if 'cmd' in data
         else '']
    )

def _host_ip():
    val = s.shell.run('ifconfig docker0|grep "inet addr:"')
    val = val.split()[1].split(':')[-1]
    return val
<commit_msg>system run can append to the cmd<commit_after>import yaml
import itertools
import sys
import argh
import s
import os

_services = './docker/state/services.yml'

@argh.arg('cmd', nargs='?', default=None)
@argh.arg('-a', '--append-to-cmd')
@argh.arg('-ncb', '--no-clean-before')
@argh.arg('-nca', '--no-clean-after')
def run(action_name, cmd, append_to_cmd=None, tty=False, no_clean_before=False, no_clean_after=False):
    """run an action"""
    build(action_name)
    tty = True if cmd == 'bash' else tty
    action = _get_action(action_name)
    if not no_clean_before:
        stop(action_name)
    try:
        _start_deps(action)
        print('start main')
        try:
            s.shell.run(_run_cmd(action['main'], tty, cmd, append_to_cmd), interactive=True)
        except:
            sys.exit(1)
    finally:
        if not no_clean_after:
            stop(action_name)

def ports(action_name, container_name=None):
    action = _get_action(action_name)
    if container_name:
        print(_exposed_port(action[container_name]['tag']))
    else:
        for container_name, data in action.items():
            port = _exposed_port(data['tag'])
            if port:
                print('{container_name}:{port}'.format(**locals()))

def stop(action_name):
    action = _get_action(action_name)
    for container_name, data in action.items():
        with s.exceptions.ignore():
            s.shell.run('sudo docker kill', data['tag'])
            s.shell.run('sudo docker rm', data['tag'])

def build(action_name, container=None, nocache=False, pull=False, force=False):
    """build an actions containers"""
    action = _get_action(action_name)
    for container_name, data in action.items():
        if not container or container_name == container:
            if force or not s.shell.run('sudo docker inspect', data['tag'], zero=True, stream=False):
                s.shell.run(_build_cmd(data, nocache, pull), stream=True)
                print(s.colors.green('built:'), data['tag'])

def show():
    """show available actions"""
    data = _get_actions()
    for action_name in sorted(data):
        print('')
        print(action_name)
        for container in data[action_name]:
            print('', container)

def main():
    assert os.path.isdir('docker'), 'must have a dir: ./docker'
    assert os.path.isfile('system.yml'), 'must have a file: ./system.yml'
    if s.shell.override('--stream'):
        with s.shell.set_stream():
            _main()
    else:
        _main()

def _main():
    s.shell.run('mkdir -p ./docker/state')
    s.shell.dispatch_commands(globals(), __name__)

def _exposed_port(tag):
    port = s.shell.run('sudo docker port', tag).split(':')[-1]
    if port:
        return int(port)

def _start_deps(action):
    s.shell.run('rm -f', _services)
    to_start = list(s.dicts.drop(action, 'main').items())
    if to_start:
        started = []
        for i in itertools.count():
            container_name, data = to_start.pop()
            deps = data.get('depends', [])
            if not deps or all(x in started for x in deps):
                print('start dep:', container_name)
                s.shell.run(_run_cmd(data, bg=True))
                started.append(container_name)
                data = {'port': _exposed_port(data['tag']),
                        'host': _host_ip()}
                with open(_services, 'a') as f:
                    f.write(yaml.dump({container_name: data}))
                s.web.wait_for_http('http://{host}:{port}'.format(**data))
            else:
                to_start.append([container_name, data])
            assert i < 500, 'never resolved dependency order. remaining: {}'.format(to_start)
            if not to_start:
                return

def _tag_containers(action_name, action):
    for container_name, data in action.items():
        action[container_name]['tag'] = '_'.join([
            os.path.basename(os.getcwd()),
            action_name,
            container_name
        ])
    return action

def _get_actions():
    with open('./system.yml') as f:
        return yaml.safe_load(f)

def _get_action(action_name):
    action = _get_actions()[action_name]
    return _tag_containers(action_name, action)

def _build_cmd(data, nocache, pull):
    return ' '.join([
        'sudo docker build',
        '-t', data['tag'],
        '--force-rm=true',
        '-f ' + data['dockerfile'] if 'dockerfile' in data else '',
        '--no-cache=true' if nocache else '--no-cache=false',
        '--pull=true' if pull else '--pull=false',
        '.',
    ])

def _run_cmd(data, tty=False, cmd=None, append_to_cmd=None, bg=False):
    append_to_cmd = append_to_cmd or ''
    return ' '.join(
        ['sudo docker run',
         '--publish-all=true',
         '-it' if tty else '',
         '-d' if bg else '',
         '--name={tag}'.format(**data),
         '-v {}/docker/state:/state'.format(os.getcwd())] +
        ['-v {}:{}'.format(os.path.abspath(os.path.expanduser(a)), b)
         for x in data.get('volumes', [])
         for y in [str(x).split(':')]
         for a, b in [(y + y)[:2]]] +
        ['-p {}:{}'.format(*z)
         for x in data.get('ports', [])
         for y in [str(x).split(':')]
         for z in [([s.net.free_port()] + y)[-2:]]] +
        [data['tag']] +
        ["bash -c '{} {}'".format(cmd, append_to_cmd)
         if cmd
         else "bash -c '{} {}'".format(data['cmd'], append_to_cmd)
         if 'cmd' in data
         else '']
    )

def _host_ip():
    val = s.shell.run('ifconfig docker0|grep "inet addr:"')
    val = val.split()[1].split(':')[-1]
    return val
<|endoftext|>


<commit_before>#!/usr/bin/env python

from s3o import S3O
from optparse import OptionParser
from glob import glob

def recursively_optimize_pieces(piece):
    optimize_piece(piece)
    for child in piece.children:
        recursively_optimize_pieces(child)

def optimize_piece(piece):
    remap = {}
    new_indices = []
    for index in piece.indices:
        vertex = piece.vertices[index]
        if vertex not in remap:
            remap[vertex] = len(remap)
        new_indices.append(remap[vertex])

    new_vertices = [(index, vertex) for vertex, index in remap.items()]
    new_vertices.sort()
    new_vertices = [vertex for index, vertex in new_vertices]

    piece.indices = new_indices
    piece.vertices = new_vertices

if __name__ == '__main__':
    parser = OptionParser(usage="%prog [options] FILES", version="%prog 0.1",
                          description="Optimize a Spring S3O file by "
                                      "removing redundant data.")
    parser.add_option("-d", "--dry-run", action="store_true",
                      default=False, dest="is_dry",
                      help="show output summary without committing changes")
    parser.add_option("-q", "--quiet", action="store_true",
                      default=False, dest="silence_output",
                      help="silence optimization summary")

    options, args = parser.parse_args()
    if len(args) < 1:
        parser.error("insufficient arguments")

    dry = options.is_dry
    silence_output = options.silence_output

    if len(args) == 1:
        filenames = glob(args[0])
    else:
        filenames = args

    for filename in filenames:
        with open(filename, 'rb+') as input_file:
            data = input_file.read()
            model = S3O(data)
            recursively_optimize_pieces(model.root_piece)
            optimized_data = model.serialize()
            delta_size = len(optimized_data) - len(data)

            if not silence_output:
                print("modified %s: "
                      "size change: %d bytes" % (filename, delta_size))

            if not dry and delta_size < 0:
                input_file.seek(0)
                input_file.truncate()
                input_file.write(optimized_data)
<commit_msg>Skip log output if delta_size > 0<commit_after>#!/usr/bin/env python

from s3o import S3O
from optparse import OptionParser
from glob import glob

def recursively_optimize_pieces(piece):
    optimize_piece(piece)
    for child in piece.children:
        recursively_optimize_pieces(child)

def optimize_piece(piece):
    remap = {}
    new_indices = []
    for index in piece.indices:
        vertex = piece.vertices[index]
        if vertex not in remap:
            remap[vertex] = len(remap)
        new_indices.append(remap[vertex])

    new_vertices = [(index, vertex) for vertex, index in remap.items()]
    new_vertices.sort()
    new_vertices = [vertex for index, vertex in new_vertices]

    piece.indices = new_indices
    piece.vertices = new_vertices

if __name__ == '__main__':
    parser = OptionParser(usage="%prog [options] FILES", version="%prog 0.1",
                          description="Optimize a Spring S3O file by "
                                      "removing redundant data.")
    parser.add_option("-d", "--dry-run", action="store_true",
                      default=False, dest="is_dry",
                      help="show output summary without committing changes")
    parser.add_option("-q", "--quiet", action="store_true",
                      default=False, dest="silence_output",
                      help="silence optimization summary")

    options, args = parser.parse_args()
    if len(args) < 1:
        parser.error("insufficient arguments")

    dry = options.is_dry
    silence_output = options.silence_output

    if len(args) == 1:
        filenames = glob(args[0])
    else:
        filenames = args

    for filename in filenames:
        with open(filename, 'rb+') as input_file:
            data = input_file.read()
            model = S3O(data)
            recursively_optimize_pieces(model.root_piece)
            optimized_data = model.serialize()
            delta_size = len(optimized_data) - len(data)

            if delta_size < 0:
                if not silence_output:
                    print("modified %s: "
                          "size change: %d bytes" % (filename, delta_size))

                if not dry:
                    input_file.seek(0)
                    input_file.truncate()
                    input_file.write(optimized_data)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import absolute_import

from salt.utils import parsers
from salt.utils import activate_profile
from salt.utils import output_profile
from salt.utils.verify import check_user, verify_log
from salt.exceptions import SaltClientError
import salt.defaults.exitcodes  # pylint: disable=W0611

class SaltRun(parsers.SaltRunOptionParser):
    '''
    Used to execute Salt runners
    '''

    def run(self):
        '''
        Execute salt-run
        '''
        import salt.runner
        self.parse_args()

        # Setup file logging!
        self.setup_logfile_logger()
        verify_log(self.config)
        profiling_enabled = self.options.profiling_enabled

        runner = salt.runner.Runner(self.config)
        if self.options.doc:
            runner.print_docs()
            self.exit(salt.defaults.exitcodes.EX_OK)

        # Run this here so SystemExit isn't raised anywhere else when
        # someone tries to use the runners via the python API
        try:
            if check_user(self.config['user']):
                pr = activate_profile(profiling_enabled)
                try:
                    ret = runner.run()
                finally:
                    output_profile(
                        pr,
                        stats_path=self.options.profiling_path,
                        stop=True)
                    if isinstance(ret, dict) and 'retcode' in ret:
                        self.exit(ret['retcode'])

        except SaltClientError as exc:
            raise SystemExit(str(exc))
<commit_msg>Fix for exception when exiting out of a runner<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import absolute_import

from salt.utils import parsers
from salt.utils import activate_profile
from salt.utils import output_profile
from salt.utils.verify import check_user, verify_log
from salt.exceptions import SaltClientError
import salt.defaults.exitcodes  # pylint: disable=W0611

class SaltRun(parsers.SaltRunOptionParser):
    '''
    Used to execute Salt runners
    '''

    def run(self):
        '''
        Execute salt-run
        '''
        import salt.runner
        self.parse_args()

        # Setup file logging!
        self.setup_logfile_logger()
        verify_log(self.config)
        profiling_enabled = self.options.profiling_enabled

        runner = salt.runner.Runner(self.config)
        if self.options.doc:
            runner.print_docs()
            self.exit(salt.defaults.exitcodes.EX_OK)

        # Run this here so SystemExit isn't raised anywhere else when
        # someone tries to use the runners via the python API
        try:
            if check_user(self.config['user']):
                pr = activate_profile(profiling_enabled)
                try:
                    ret = runner.run()
                    if isinstance(ret, dict) and 'retcode' in ret:
                        self.exit(ret['retcode'])
                finally:
                    output_profile(
                        pr,
                        stats_path=self.options.profiling_path,
                        stop=True)

        except SaltClientError as exc:
            raise SystemExit(str(exc))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Many aspects of the salt payload need to be managed, from the return of
encrypted keys to general payload dynamics and packaging, these happen
in here
'''

# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
# import sys  # Use if sys is commented out below
import logging
import gc
import datetime

# Import salt libs
import salt.log
import salt.transport.frame
import salt.utils.immutabletypes as immutabletypes
import salt.utils.stringutils
from salt.exceptions import SaltReqTimeoutError
from salt.utils.data import CaseInsensitiveDict

# Import third party libs
from salt.ext import six
try:
    import zmq
except ImportError:
    # No need for zeromq in local mode
    pass

log = logging.getLogger(__name__)

HAS_MSGPACK = False
try:
    # Attempt to import msgpack
    import msgpack
    # There is a serialization issue on ARM and potentially other platforms
    # for some msgpack bindings, check for it
    if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None:
        raise ImportError
    HAS_MSGPACK = True
except ImportError:
    # Fall back to msgpack_pure
    try:
        import msgpack_pure as msgpack  # pylint: disable=import-error
        HAS_MSGPACK = True
    except ImportError:
        # TODO: Come up with a sane way to get a configured logfile
        #       and write to the logfile when this error is hit also
        LOG_FORMAT = '[%(levelname)-8s] %(message)s'
        salt.log.setup_console_logger(log_format=LOG_FORMAT)
        log.fatal('Unable to import msgpack or msgpack_pure python modules')
        # Don't exit if msgpack is not available, this is to make local mode
        # work without msgpack
        #sys.exit(salt.defaults.exitcodes.EX_GENERIC)

if HAS_MSGPACK and not hasattr(msgpack, 'exceptions'):
    class PackValueError(Exception):
        '''
        older versions of msgpack do not have PackValueError
        '''

    class exceptions(object):
        '''
        older versions of msgpack do not have an exceptions module
        '''
        PackValueError = PackValueError()

    msgpack.exceptions = exceptions()

def package(payload):
    '''
    This method for now just wraps msgpack.dumps, but it is here so that
    we can make the serialization a custom option in the future with ease.
    '''
    return msgpack.dumps(payload)

def unpackage(package_):
    '''
    Unpackages a payload
    '''
    return msgpack.loads(package_, use_list=True)

def format_payload(enc, **kwargs):
    '''
    Pass in the required arguments for a payload, the enc type and the cmd,
    then a list of keyword args to generate the body of the load dict.
    '''
    payload = {'enc': enc}
    load = {}
    for key in kwargs:
        load[key] = kwargs[key]
    payload['load'] = load
    return package(payload)

class Serial(object):
    '''
    Create a serialization object, this object manages all message
    serialization in Salt
    '''
    def __init__(self, opts):
        if isinstance(opts, dict):
            self.serial = opts.get('serial', 'msgpack')
        elif isinstance(opts, six.string_types):
            self.serial = opts
        else:
            self.serial = 'msgpack'

    def loads(self, msg, encoding=None, raw=False):
        '''
        Run the correct loads serialization format

        :param encoding: Useful for Python 3 support. If the msgpack data
                         was encoded using "use_bin_type=True", this will
                         differentiate between the 'bytes' type and the
                         'str' type by decoding contents with 'str' type
                         to what the encoding was set as. Recommended
                         encoding is 'utf-8' when using Python 3.
                         If the msgpack data was not encoded using
                         "use_bin_type=True", it will try to decode
                         all 'bytes' and 'str' data (the distinction has
                         been lost in this case) to what the encoding is
                         set as. In this case, it will fail if any of
                         the contents cannot be converted.
        '''
        try:
            def ext_type_decoder(code, data):
                if code == 78:
                    data = salt.utils.stringutils.to_unicode(data)
                    return datetime.datetime.strptime(data, '%Y%m%dT%H:%M:%S.%f')
                return data

            gc.disable()  # performance optimization for msgpack
            if msgpack.version >= (0, 4, 0):
                # msgpack only supports 'encoding' starting in 0.4.0.
                # Due to this, if we don't need it, don't pass it at all so
                # that under Python 2 we can still work with older versions
                # of msgpack.
                try:
                    ret = msgpack.loads(msg, use_list=True, ext_hook=ext_type_decoder, encoding=encoding)
                except UnicodeDecodeError:
                    # msg contains binary data
                    ret = msgpack.loads(msg, use_list=True, ext_hook=ext_type_decoder)
            else:
                ret = msgpack.loads(msg, use_list=True, ext_hook=ext_type_decoder)
            if six.PY3 and encoding is None and not raw:
                ret = salt.transport.frame.decode_embedded_strs(ret)
        except Exception as exc:
            log.critical(
                'Could not deserialize msgpack message. This often happens '
                'when trying to read a file not in binary mode. '
                'To see message payload, enable debug logging and retry. '
                'Exception: %s', exc
            )
            log.debug('Msgpack deserialization failure on message: %s', msg)
            gc.collect()
            raise
        finally:
            gc.enable()
        return ret

    def load(self, fn_):
        '''
        Run the correct serialization to load a file
        '''
        data = fn_.read()
        fn_.close()
        if data:
            if six.PY3:
                return self.loads(data, encoding='utf-8')
            else:
                return self.loads(data)

    def dumps(self, msg, use_bin_type=False):
        '''
        Run the correct dumps serialization format

        :param use_bin_type: Useful for Python 3 support. Tells msgpack to
                             differentiate between 'str' and 'bytes' types
                             by encoding them differently.
                             Since this changes the wire protocol, this
                             option should not be used outside of IPC.
        '''
        def ext_type_encoder(obj):
            if isinstance(obj, six.integer_types):
                # msgpack can't handle the very long Python longs for jids
                # Convert any very long longs to strings
                return six.text_type(obj)
            elif isinstance(obj, datetime.datetime):
                # msgpack doesn't support datetime.datetime datatype
                # So here we have converted datetime.datetime to custom datatype
                # This is msgpack Extended types numbered 78
                return msgpack.ExtType(78, salt.utils.stringutils.to_bytes(
                    obj.strftime('%Y%m%dT%H:%M:%S.%f')))
            # The same for immutable types
            elif isinstance(obj, immutabletypes.ImmutableDict):
                return dict(obj)
            elif isinstance(obj, immutabletypes.ImmutableList):
                return list(obj)
            elif isinstance(obj, (set, immutabletypes.ImmutableSet)):
                # msgpack can't handle set so translate it to tuple
                return tuple(obj)
            elif isinstance(obj, CaseInsensitiveDict):
                return dict(obj)
            # Nothing known exceptions found. Let msgpack raise it's own.
            return obj

        try:
            if msgpack.version >= (0, 4, 0):
                # msgpack only supports 'use_bin_type' starting in 0.4.0.
                # Due to this, if we don't need it, don't pass it at all so
                # that under Python 2 we can still work with older versions
                # of msgpack.
                return msgpack.dumps(msg, default=ext_type_encoder, use_bin_type=use_bin_type)
            else:
                return msgpack.dumps(msg, default=ext_type_encoder)
        except (OverflowError, msgpack.exceptions.PackValueError):
            # msgpack<=0.4.6 don't call ext encoder on very long integers raising the error instead.
            # Convert any very long longs to strings and call dumps again.
            def verylong_encoder(obj):
                if isinstance(obj, dict):
                    for key, value in six.iteritems(obj.copy()):
                        obj[key] = verylong_encoder(value)
                    return dict(obj)
                elif isinstance(obj, (list, tuple)):
                    obj = list(obj)
                    for idx, entry in enumerate(obj):
                        obj[idx] = verylong_encoder(entry)
                    return obj
                # A value of an Integer object is limited from -(2^63) upto (2^64)-1 by MessagePack
                # spec. Here we care only of JIDs that are positive integers.
                if isinstance(obj, six.integer_types) and obj >= pow(2, 64):
                    return six.text_type(obj)
                else:
                    return obj

            msg = verylong_encoder(msg)
            if msgpack.version >= (0, 4, 0):
                return msgpack.dumps(msg, default=ext_type_encoder, use_bin_type=use_bin_type)
            else:
                return msgpack.dumps(msg, default=ext_type_encoder)

    def dump(self, msg, fn_):
        '''
        Serialize the correct data into the named file object
        '''
        if six.PY2:
            fn_.write(self.dumps(msg))
        else:
            # When using Python 3, write files in such a way
            # that the 'bytes' and 'str' types are distinguishable
            # by using "use_bin_type=True".
            fn_.write(self.dumps(msg, use_bin_type=True))
        fn_.close()

class SREQ(object):
    '''
    Create a generic interface to wrap salt zeromq req calls.
    '''
    def __init__(self, master, id_='', serial='msgpack', linger=0, opts=None):
        self.master = master
        self.id_ = id_
        self.serial = Serial(serial)
        self.linger = linger
        self.context = zmq.Context()
        self.poller = zmq.Poller()
        self.opts = opts

    @property
    def socket(self):
        '''
        Lazily create the socket.
        '''
        if not hasattr(self, '_socket'):
            # create a new one
            self._socket = self.context.socket(zmq.REQ)
            if hasattr(zmq, 'RECONNECT_IVL_MAX'):
                self._socket.setsockopt(
                    zmq.RECONNECT_IVL_MAX, 5000
                )

            self._set_tcp_keepalive()
            if self.master.startswith('tcp://['):
                # Hint PF type if bracket enclosed IPv6 address
                if hasattr(zmq, 'IPV6'):
                    self._socket.setsockopt(zmq.IPV6, 1)
                elif hasattr(zmq, 'IPV4ONLY'):
                    self._socket.setsockopt(zmq.IPV4ONLY, 0)
            self._socket.linger = self.linger
            if self.id_:
                self._socket.setsockopt(zmq.IDENTITY, self.id_)
            self._socket.connect(self.master)
        return self._socket

    def _set_tcp_keepalive(self):
        if hasattr(zmq, 'TCP_KEEPALIVE') and self.opts:
            if 'tcp_keepalive' in self.opts:
                self._socket.setsockopt(
                    zmq.TCP_KEEPALIVE, self.opts['tcp_keepalive']
                )
            if 'tcp_keepalive_idle' in self.opts:
                self._socket.setsockopt(
                    zmq.TCP_KEEPALIVE_IDLE, self.opts['tcp_keepalive_idle']
                )
            if 'tcp_keepalive_cnt' in self.opts:
                self._socket.setsockopt(
                    zmq.TCP_KEEPALIVE_CNT, self.opts['tcp_keepalive_cnt']
                )
            if 'tcp_keepalive_intvl' in self.opts:
                self._socket.setsockopt(
                    zmq.TCP_KEEPALIVE_INTVL, self.opts['tcp_keepalive_intvl']
                )

    def clear_socket(self):
        '''
        delete socket if you have it
        '''
        if hasattr(self, '_socket'):
            if isinstance(self.poller.sockets, dict):
                sockets = list(self.poller.sockets.keys())
                for socket in sockets:
                    log.trace('Unregistering socket: %s', socket)
                    self.poller.unregister(socket)
            else:
                for socket in self.poller.sockets:
                    log.trace('Unregistering socket: %s', socket)
                    self.poller.unregister(socket[0])
            del self._socket

    def send(self, enc, load, tries=1, timeout=60):
        '''
        Takes two arguments, the encryption type and the base payload
        '''
        payload = {'enc': enc}
        payload['load'] = load
        pkg = self.serial.dumps(payload)
        self.socket.send(pkg)
        self.poller.register(self.socket, zmq.POLLIN)
        tried = 0
        while True:
            polled = self.poller.poll(timeout * 1000)
            tried += 1
            if polled:
                break
            if tries > 1:
                log.info(
                    'SaltReqTimeoutError: after %s seconds. (Try %s of %s)',
                    timeout, tried, tries
                )
            if tried >= tries:
                self.clear_socket()
                raise SaltReqTimeoutError(
                    'SaltReqTimeoutError: after {0} seconds, ran {1} '
                    'tries'.format(timeout * tried, tried)
                )
        return self.serial.loads(self.socket.recv())

    def send_auto(self, payload, tries=1, timeout=60):
        '''
        Detect the encryption type based on the payload
        '''
        enc = payload.get('enc', 'clear')
        load = payload.get('load', {})
        return self.send(enc, load, tries, timeout)

    def destroy(self):
        if isinstance(self.poller.sockets, dict):
            sockets = list(self.poller.sockets.keys())
            for socket in sockets:
                if socket.closed is False:
                    socket.setsockopt(zmq.LINGER, 1)
                    socket.close()
                self.poller.unregister(socket)
        else:
            for socket in self.poller.sockets:
                if socket[0].closed is False:
                    socket[0].setsockopt(zmq.LINGER, 1)
                    socket[0].close()
                self.poller.unregister(socket[0])
        if self.socket.closed is False:
            self.socket.setsockopt(zmq.LINGER, 1)
            self.socket.close()
        if self.context.closed is False:
            self.context.term()

    def __del__(self):
        self.destroy()
<commit_msg>Fix deprecation warning in msgpack >= 0.5.2<commit_after># -*- coding: utf-8 -*-
'''
Many aspects of the salt payload need to be managed, from the return of
encrypted keys to general payload dynamics and packaging, these happen
in here
'''

# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
# import sys  # Use if sys is commented out below
import logging
import gc
import datetime

# Import salt libs
import salt.log
import salt.transport.frame
import salt.utils.immutabletypes as immutabletypes
import salt.utils.stringutils
from salt.exceptions import SaltReqTimeoutError
from salt.utils.data import CaseInsensitiveDict

# Import third party libs
from salt.ext import six
try:
    import zmq
except ImportError:
    # No need for zeromq in local mode
    pass

log = logging.getLogger(__name__)

HAS_MSGPACK = False
try:
    # Attempt to import msgpack
    import msgpack
    # There is a serialization issue on ARM and potentially other platforms
    # for some msgpack bindings, check for it
    if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None:
        raise ImportError
    HAS_MSGPACK = True
except ImportError:
    # Fall back to msgpack_pure
    try:
        import msgpack_pure as msgpack  # pylint: disable=import-error
        HAS_MSGPACK = True
    except ImportError:
        # TODO: Come up with a sane way to get a configured logfile
        #       and write to the logfile when this error is hit also
        LOG_FORMAT = '[%(levelname)-8s] %(message)s'
        salt.log.setup_console_logger(log_format=LOG_FORMAT)
        log.fatal('Unable to import msgpack or msgpack_pure python modules')
        # Don't exit if msgpack is not available, this is to make local mode
        # work without msgpack
        #sys.exit(salt.defaults.exitcodes.EX_GENERIC)

if HAS_MSGPACK and not hasattr(msgpack, 'exceptions'):
    class PackValueError(Exception):
        '''
        older versions of msgpack do not have PackValueError
        '''

    class exceptions(object):
        '''
        older versions of msgpack do not have an exceptions module
        '''
        PackValueError = PackValueError()

    msgpack.exceptions = exceptions()

def package(payload):
    '''
    This method for now just wraps msgpack.dumps, but it is here so that
    we can make the serialization a custom option in the future with ease.
    '''
    return msgpack.dumps(payload)

def unpackage(package_):
    '''
    Unpackages a payload
    '''
    return msgpack.loads(package_, use_list=True)

def format_payload(enc, **kwargs):
    '''
    Pass in the required arguments for a payload, the enc type and the cmd,
    then a list of keyword args to generate the body of the load dict.
    '''
    payload = {'enc': enc}
    load = {}
    for key in kwargs:
        load[key] = kwargs[key]
    payload['load'] = load
    return package(payload)

class Serial(object):
    '''
    Create a serialization object, this object manages all message
    serialization in Salt
    '''
    def __init__(self, opts):
        if isinstance(opts, dict):
            self.serial = opts.get('serial', 'msgpack')
        elif isinstance(opts, six.string_types):
            self.serial = opts
        else:
            self.serial = 'msgpack'

    def loads(self, msg, encoding=None, raw=False):
        '''
        Run the correct loads serialization format

        :param encoding: Useful for Python 3 support. If the msgpack data
                         was encoded using "use_bin_type=True", this will
                         differentiate between the 'bytes' type and the
                         'str' type by decoding contents with 'str' type
                         to what the encoding was set as. Recommended
                         encoding is 'utf-8' when using Python 3.
                         If the msgpack data was not encoded using
                         "use_bin_type=True", it will try to decode
                         all 'bytes' and 'str' data (the distinction has
                         been lost in this case) to what the encoding is
                         set as. In this case, it will fail if any of
                         the contents cannot be converted.
        '''
        try:
            def ext_type_decoder(code, data):
                if code == 78:
                    data = salt.utils.stringutils.to_unicode(data)
                    return datetime.datetime.strptime(data, '%Y%m%dT%H:%M:%S.%f')
                return data

            gc.disable()  # performance optimization for msgpack
            loads_kwargs = {'use_list': True,
                            'ext_hook': ext_type_decoder}
            if msgpack.version >= (0, 4, 0):
                # msgpack only supports 'encoding' starting in 0.4.0.
                # Due to this, if we don't need it, don't pass it at all so
                # that under Python 2 we can still work with older versions
                # of msgpack.
                if msgpack_version >= (0, 5, 2):
                    loads_kwargs['raw'] = False
                else:
                    loads_kwargs['encoding'] = encoding
                try:
                    ret = msgpack.loads(msg, **loads_kwargs)
                except UnicodeDecodeError:
                    # msg contains binary data
                    loads_kwargs.pop('raw', None)
                    loads_kwargs.pop('encoding', None)
                    ret = msgpack.loads(msg, **loads_kwargs)
            else:
                ret = msgpack.loads(msg, **loads_kwargs)
            if six.PY3 and encoding is None and not raw:
                ret = salt.transport.frame.decode_embedded_strs(ret)
        except Exception as exc:
            log.critical(
                'Could not deserialize msgpack message. This often happens '
                'when trying to read a file not in binary mode. '
                'To see message payload, enable debug logging and retry. '
                'Exception: %s', exc
            )
            log.debug('Msgpack deserialization failure on message: %s', msg)
            gc.collect()
            raise
        finally:
            gc.enable()
        return ret

    def load(self, fn_):
        '''
        Run the correct serialization to load a file
        '''
        data = fn_.read()
        fn_.close()
        if data:
            if six.PY3:
                return self.loads(data, encoding='utf-8')
            else:
                return self.loads(data)

    def dumps(self, msg, use_bin_type=False):
        '''
        Run the correct dumps serialization format

        :param use_bin_type: Useful for Python 3 support. Tells msgpack to
                             differentiate between 'str' and 'bytes' types
                             by encoding them differently.
                             Since this changes the wire protocol, this
                             option should not be used outside of IPC.
        '''
        def ext_type_encoder(obj):
            if isinstance(obj, six.integer_types):
                # msgpack can't handle the very long Python longs for jids
                # Convert any very long longs to strings
                return six.text_type(obj)
            elif isinstance(obj, datetime.datetime):
                # msgpack doesn't support datetime.datetime datatype
                # So here we have converted datetime.datetime to custom datatype
                # This is msgpack Extended types numbered 78
                return msgpack.ExtType(78, salt.utils.stringutils.to_bytes(
                    obj.strftime('%Y%m%dT%H:%M:%S.%f')))
            # The same for immutable types
            elif isinstance(obj, immutabletypes.ImmutableDict):
                return dict(obj)
            elif isinstance(obj, immutabletypes.ImmutableList):
                return list(obj)
            elif isinstance(obj, (set, immutabletypes.ImmutableSet)):
                # msgpack can't handle set so translate it to tuple
                return tuple(obj)
            elif isinstance(obj, CaseInsensitiveDict):
                return dict(obj)
            # Nothing known exceptions found. Let msgpack raise it's own.
            return obj

        try:
            if msgpack.version >= (0, 4, 0):
                # msgpack only supports 'use_bin_type' starting in 0.4.0.
                # Due to this, if we don't need it, don't pass it at all so
                # that under Python 2 we can still work with older versions
                # of msgpack.
                return msgpack.dumps(msg, default=ext_type_encoder, use_bin_type=use_bin_type)
            else:
                return msgpack.dumps(msg, default=ext_type_encoder)
        except (OverflowError, msgpack.exceptions.PackValueError):
            # msgpack<=0.4.6 don't call ext encoder on very long integers raising the error instead.
            # Convert any very long longs to strings and call dumps again.
            def verylong_encoder(obj):
                if isinstance(obj, dict):
                    for key, value in six.iteritems(obj.copy()):
                        obj[key] = verylong_encoder(value)
                    return dict(obj)
                elif isinstance(obj, (list, tuple)):
                    obj = list(obj)
                    for idx, entry in enumerate(obj):
                        obj[idx] = verylong_encoder(entry)
                    return obj
                # A value of an Integer object is limited from -(2^63) upto (2^64)-1 by MessagePack
                # spec. Here we care only of JIDs that are positive integers.
                if isinstance(obj, six.integer_types) and obj >= pow(2, 64):
                    return six.text_type(obj)
                else:
                    return obj

            msg = verylong_encoder(msg)
            if msgpack.version >= (0, 4, 0):
                return msgpack.dumps(msg, default=ext_type_encoder, use_bin_type=use_bin_type)
            else:
                return msgpack.dumps(msg, default=ext_type_encoder)

    def dump(self, msg, fn_):
        '''
        Serialize the correct data into the named file object
        '''
        if six.PY2:
            fn_.write(self.dumps(msg))
        else:
            # When using Python 3, write files in such a way
            # that the 'bytes' and 'str' types are distinguishable
            # by using "use_bin_type=True".
            fn_.write(self.dumps(msg, use_bin_type=True))
        fn_.close()

class SREQ(object):
    '''
    Create a generic interface to wrap salt zeromq req calls.
    '''
    def __init__(self, master, id_='', serial='msgpack', linger=0, opts=None):
        self.master = master
        self.id_ = id_
        self.serial = Serial(serial)
        self.linger = linger
        self.context = zmq.Context()
        self.poller = zmq.Poller()
        self.opts = opts

    @property
    def socket(self):
        '''
        Lazily create the socket.
        '''
        if not hasattr(self, '_socket'):
            # create a new one
            self._socket = self.context.socket(zmq.REQ)
            if hasattr(zmq, 'RECONNECT_IVL_MAX'):
                self._socket.setsockopt(
                    zmq.RECONNECT_IVL_MAX, 5000
                )

            self._set_tcp_keepalive()
            if self.master.startswith('tcp://['):
                # Hint PF type if bracket enclosed IPv6 address
                if hasattr(zmq, 'IPV6'):
                    self._socket.setsockopt(zmq.IPV6, 1)
                elif hasattr(zmq, 'IPV4ONLY'):
                    self._socket.setsockopt(zmq.IPV4ONLY, 0)
            self._socket.linger = self.linger
            if self.id_:
                self._socket.setsockopt(zmq.IDENTITY, self.id_)
            self._socket.connect(self.master)
        return self._socket

    def _set_tcp_keepalive(self):
        if hasattr(zmq, 'TCP_KEEPALIVE') and self.opts:
            if 'tcp_keepalive' in self.opts:
                self._socket.setsockopt(
                    zmq.TCP_KEEPALIVE, self.opts['tcp_keepalive']
                )
            if 'tcp_keepalive_idle' in self.opts:
                self._socket.setsockopt(
                    zmq.TCP_KEEPALIVE_IDLE, self.opts['tcp_keepalive_idle']
                )
            if 'tcp_keepalive_cnt' in self.opts:
                self._socket.setsockopt(
                    zmq.TCP_KEEPALIVE_CNT, self.opts['tcp_keepalive_cnt']
                )
            if 'tcp_keepalive_intvl' in self.opts:
                self._socket.setsockopt(
                    zmq.TCP_KEEPALIVE_INTVL, self.opts['tcp_keepalive_intvl']
                )

    def clear_socket(self):
        '''
        delete socket if you have it
        '''
        if hasattr(self, '_socket'):
            if isinstance(self.poller.sockets, dict):
                sockets = list(self.poller.sockets.keys())
                for socket in sockets:
                    log.trace('Unregistering socket: %s', socket)
                    self.poller.unregister(socket)
            else:
                for socket in self.poller.sockets:
                    log.trace('Unregistering socket: %s', socket)
                    self.poller.unregister(socket[0])
            del self._socket

    def send(self, enc, load, tries=1, timeout=60):
        '''
        Takes two arguments, the encryption type and the base payload
        '''
        payload = {'enc': enc}
        payload['load'] = load
        pkg = self.serial.dumps(payload)
        self.socket.send(pkg)
        self.poller.register(self.socket, zmq.POLLIN)
        tried = 0
        while True:
            polled = self.poller.poll(timeout * 1000)
            tried += 1
            if polled:
                break
            if tries > 1:
                log.info(
                    'SaltReqTimeoutError: after %s seconds. (Try %s of %s)',
                    timeout, tried, tries
                )
            if tried >= tries:
                self.clear_socket()
                raise SaltReqTimeoutError(
                    'SaltReqTimeoutError: after {0} seconds, ran {1} '
                    'tries'.format(timeout * tried, tried)
                )
        return self.serial.loads(self.socket.recv())

    def send_auto(self, payload, tries=1, timeout=60):
        '''
        Detect the encryption type based on the payload
        '''
        enc = payload.get('enc', 'clear')
        load = payload.get('load', {})
        return self.send(enc, load, tries, timeout)

    def destroy(self):
        if isinstance(self.poller.sockets, dict):
            sockets = list(self.poller.sockets.keys())
            for socket in sockets:
                if socket.closed is False:
                    socket.setsockopt(zmq.LINGER, 1)
                    socket.close()
                self.poller.unregister(socket)
        else:
            for socket in self.poller.sockets:
                if socket[0].closed is False:
                    socket[0].setsockopt(zmq.LINGER, 1)
                    socket[0].close()
                self.poller.unregister(socket[0])
        if self.socket.closed is False:
            self.socket.setsockopt(zmq.LINGER, 1)
            self.socket.close()
        if self.context.closed is False:
            self.context.term()

    def __del__(self):
        self.destroy()
<|endoftext|>


<commit_before># coding: utf-8

from tapioca import (
    TapiocaAdapter, generate_wrapper_from_adapter, JSONAdapterMixin)
from resource_mapping import RESOURCE_MAPPING

class InstagramClientAdapter(JSONAdapterMixin, TapiocaAdapter):
    api_root = 'https://api.instagram.com/v1/'
    resource_mapping = RESOURCE_MAPPING

    def get_request_kwargs(self, api_params, *args, **kwargs):
        params = super(InstagramClientAdapter, self).get_request_kwargs(
            api_params, *args, **kwargs)
        params['params'] = {
            'access_token': api_params.get('access_token', '')
        }
        return params

    def get_iterator_list(self, response_data):
        return response_data

    def get_iterator_next_request_kwargs(self,
            iterator_request_kwargs, response_data, response):
        pass

Instagram = generate_wrapper_from_adapter(InstagramClientAdapter)
<commit_msg>Added pagination<commit_after># coding: utf-8

from tapioca import (
    TapiocaAdapter, generate_wrapper_from_adapter, JSONAdapterMixin)
from resource_mapping import RESOURCE_MAPPING

class InstagramClientAdapter(JSONAdapterMixin, TapiocaAdapter):
    api_root = 'https://api.instagram.com/v1/'
    resource_mapping = RESOURCE_MAPPING

    def get_request_kwargs(self, api_params, *args, **kwargs):
        params = super(InstagramClientAdapter, self).get_request_kwargs(
            api_params, *args, **kwargs)
        params['params'] = {
            'access_token': api_params.get('access_token', '')
        }
        return params

    def get_iterator_list(self, response_data):
        return response_data['data']

    def get_iterator_next_request_kwargs(self,
            iterator_request_kwargs, response_data, response):
        paging = response_data.get('pagination')
        if not paging:
            return
        url = paging.get('next_url')
        if url:
            return {'url': url}

Instagram = generate_wrapper_from_adapter(InstagramClientAdapter)
<|endoftext|>