<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import argparse
import io
import logging
import sys
import requests
import urllib3
import tld
from tld import get_tld
from bs4 import BeautifulSoup
import html
from PyPDF2 import PdfFileReader, PdfFileWriter

def main():
    # See https://blog.quora.com/Launched-Customizable-Links for Quora's
    # launch post
    parser = argparse.ArgumentParser(description=("Get the linked title of " +
        "URLs (similar to Quora and Facebook)"))
    parser.add_argument("url", type=str, help="the URL")
    parser.add_argument("-f", "--format", type=str,
            help=("output format; accepted values are 'none', 'html', " +
            "'markdown', 'latex', 'mediawiki'"))
    parser.add_argument("-c", "--clean", action="store_true",
            help=("clean the title to remove the site name " +
            "if the title was obtained from an HTML title tag"))
    parser.add_argument("-v", "--verbose", action="store_const",
        dest="log_level", const=logging.DEBUG, help="enable debug messages")
    args = parser.parse_args()
    logging.basicConfig(level=args.log_level)

    url = args.url
    logging.debug("Trying first attempt")
    attempt_1 = try_url(url, args.format, clean=args.clean)
    if attempt_1["exit"]:
        logging.debug("First attempt succeeded!")
        print(attempt_1["text"], end="")
    else:
        logging.debug("First attempt failed; trying second attempt")
        attempt_2 = try_url("http://" + url, args.format, clean=args.clean)
        if attempt_2["exit"]:
            logging.debug("Second attempt succeeded!")
            print(attempt_2["text"], end="")
        else:
            logging.debug("Second attempted failed; using result " +
                "from first attempt")
            print(attempt_1["text"], end="")

def try_url(url, fmt, clean=False):
    '''
    Return (Str, True) if succeeded; (Str, False) otherwise.
    '''
    result = {}
    analyte = analyze_url(url)
    if analyte:
        return {"text": analyte, "exit": True}
    try:
        user_agent = ("Mozilla/5.0 (X11; Linux i686; rv:31.0) Gecko/20100101 " +
            "Firefox/31.0 Iceweasel/31.5.0")
        headers = {"User-Agent": user_agent}
        response = requests.get(url, stream=True, headers=headers)
        url = response.url
        if  ("text/html" in response.headers["content-type"] or
                "application/pdf" in response.headers["content-type"]):
            logging.debug("HTML page or PDF file detected")
            # <title> is probably in the first around 10MB
            doc = response.iter_content(chunk_size=1000000)
            data = next(doc)
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"], data=data,
                    clean=clean),
                url,
                fmt
            )
        else:
            logging.debug("No HTML page detected")
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"]),
                url,
                fmt
            )
        result["exit"] = True
    except (requests.exceptions.MissingSchema,
            requests.exceptions.ConnectionError,
            requests.exceptions.InvalidSchema,
            urllib3.exceptions.LocationParseError):
        # since it's not a valid URL, just return it
        result["text"] = url
        result["exit"] = False
    return result

def analyze_url(url):
    """
    Look just at the URL to see if a suitable title text can be found.  This
    method is much faster than actually visiting the URL to find the title
    element in the downloaded file. We want to do this for special sites like
    Facebook, which doesn't allow anonymous downloading of certain pages, like
    group pages.

    Args:
        url: A string that is a URL

    Returns:
        A string that is the title text to be used. If no suitable title text
        can be produced, return the empty string, "".
    """
    try:
        tl = get_tld(url)
    except (tld.exceptions.TldBadUrl, tld.exceptions.TldDomainNotFound):
        logging.debug("bad TLD; trying with http:// prefix")
        try:
            tl = get_tld("http://" + url)
        except (tld.exceptions.TldBadUrl, tld.exceptions.TldDomainNotFound):
            logging.debug("still bad TLD; giving up")
            return ""
    if tl == "facebook.com" and "facebook.com/groups/" in url:
            return "Facebook group page post"
    return ""

def get_filetype_link(link_text, url, filetype):
    """
    Put the pieces together and produce a valid hyperlink for given output
    filetype.

    Args:
        link_text: A string representing the displayed or linked text when
            linking to something, e.g. "hello" in hello. This string should already be in
            the intended form; i.e. all HTML escapes should have been unescaped
            at this point.
        url: A string of the URL.
        filetype: A string of the output filetype. Accepted parameters are:
            "none", "html", "markdown", "latex".

    Returns:
        A string that is a valid hyperlink for the specified output filetype.
    """
    if filetype == "markdown":
        # From http://pandoc.org/README.html#backslash-escapes
        special_chars = "\\`*_{}[]()>#+-.!"
        result = ""
        for c in link_text:
            if c in special_chars:
                result += "\\" + c
            else:
                result += c
        return "[{link_text}]({url})".format(link_text=result, url=url)
    if filetype == "html":
        return '{link_text}'.format(url=url,
                link_text=html.escape(link_text))
    if filetype == "mediawiki":
        return "[{url} {link_text}]".format(url=url,
                link_text=link_text)
    if filetype == "latex":
        # LaTeX is really sensitive about special characters so this
        # probably needs a lot of tweaking
        special_chars = "$&%{_#"
        result = ""
        for c in link_text:
            if c in special_chars:
                result += "\\" + c
            elif c == "\\":
                result += "\\textbackslash{}"
            elif c == "~":
                result += "\\textasciitilde{}"
            else:
                result += c
        clean_url = ""
        for c in url:
            if c in special_chars or c in "~":
                clean_url += "\\" + c
            elif c == "\\":
                clean_url += "{\\textbackslash}"
            else:
                clean_url += c
        return ("\\href{%s}{%s}" % (clean_url, result))
    else:
        return "{link_text}: {url}".format(url=url, link_text=link_text)

def get_link_text(url, mime_type, data=None, clean=False):
    '''
    Take URL, MIME type, and optional data to produce the link text.
    '''
    tld = get_tld(url)
    result = "File on " + tld
    if mime_type.startswith("image"):
        result = "Image on " + tld
    elif  "application/pdf" in mime_type:
        logging.debug("PDF detected")
        # I need seek() for some reason so convert from bytes
        data = io.BytesIO(data)
        # fix this later, but I always get a "PdfReadWarning: Xref table
        # not zero-indexed" which should only happen when the -v flag is
        # present
        import warnings
        warnings.filterwarnings("ignore")
        pdf = PdfFileReader(data, strict=True)
        # PyPDF2 somehow thinks many PDFs are encrypted with the empty
        # string, so deal with that
        if pdf.isEncrypted:
            pdf.decrypt('')
        result = pdf.getDocumentInfo().title
        #result = "PDF on " + tld
    elif "text/html" in mime_type:
        try:
            soup = BeautifulSoup(data, 'html.parser')
            meta = soup.find_all("meta")
            og_title_lst = []
            twitter_title_lst = []
            meta_title_lst = []
            schema_lst = []
            for i in meta:
                if i.get("property") == "og:title":
                    og_title_lst.append(i.get("content"))
                elif i.get("property") == "twitter:title":
                    twitter_title_lst.append(i.get("content"))
                elif i.get("name") == "title":
                    meta_title_lst.append(i.get("content"))
                elif i.get("itemprop") == "name":
                    schema_lst.append(i.get("content"))
            if og_title_lst:
                logging.debug("found og:title")
                result = og_title_lst[0].strip()
            elif twitter_title_lst:
                logging.debug("found twitter title")
                result = twitter_title_lst[0].strip()
            elif meta_title_lst:
                logging.debug("found meta name title")
                result = meta_title_lst[0].strip()
            elif schema_lst:
                logging.debug("found schema title")
                result = schema_lst[0].strip()
            elif soup.title and soup.title.string:
                logging.debug("found title tag")
                result = html.unescape(soup.title.string)
                if clean:
                    result = messy_title_parse(result)
            else:
                logging.debug("no title found; using default")
                result = "Page on " + tld
        except AttributeError:
            # Probably just empty title when trying to get
            # soup.title.string
            logging.debug("FIXME: this isn't supposed to happen")
            result = "Page on " + tld
    if len(result) > 255:
        result = result[:253] + " …"

    return result

def messy_title_parse(title, url=None):
    # Even if nothing works, at least we'll have a whitespace-sanitized
    # title
    logging.debug("Cleaning messy title")
    result = title.strip()
    hyphen_split = result.split(" - ")
    bar_split = result.split(" | ")
    em_dash_split = result.split(" — ")
    colon_split = result.split(": ")
    if len(hyphen_split) > 1:
        # So there is actually more than one part, so we just take the
        # first and we're done.  This is for titles like "Post Title -
        # Site Name"
        result = hyphen_split[0]
    elif len(em_dash_split) > 1:
        result = em_dash_split[0]
    elif len(bar_split) > 1:
        result = bar_split[0]
    elif len(colon_split) > 1:
        # For titles like "Site Name: Post Title"
        result = colon_split[-1]
    return result

if __name__ == "__main__":
    main()
<commit_msg>add fixme note<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import argparse
import io
import logging
import sys
import requests
import urllib3
import tld
from tld import get_tld
from bs4 import BeautifulSoup
import html
from PyPDF2 import PdfFileReader, PdfFileWriter

def main():
    # See https://blog.quora.com/Launched-Customizable-Links for Quora's
    # launch post
    parser = argparse.ArgumentParser(description=("Get the linked title of " +
        "URLs (similar to Quora and Facebook)"))
    parser.add_argument("url", type=str, help="the URL")
    parser.add_argument("-f", "--format", type=str,
            help=("output format; accepted values are 'none', 'html', " +
            "'markdown', 'latex', 'mediawiki'"))
    parser.add_argument("-c", "--clean", action="store_true",
            help=("clean the title to remove the site name " +
            "if the title was obtained from an HTML title tag"))
    parser.add_argument("-v", "--verbose", action="store_const",
        dest="log_level", const=logging.DEBUG, help="enable debug messages")
    args = parser.parse_args()
    logging.basicConfig(level=args.log_level)

    url = args.url
    logging.debug("Trying first attempt")
    attempt_1 = try_url(url, args.format, clean=args.clean)
    if attempt_1["exit"]:
        logging.debug("First attempt succeeded!")
        print(attempt_1["text"], end="")
    else:
        logging.debug("First attempt failed; trying second attempt")
        attempt_2 = try_url("http://" + url, args.format, clean=args.clean)
        if attempt_2["exit"]:
            logging.debug("Second attempt succeeded!")
            print(attempt_2["text"], end="")
        else:
            logging.debug("Second attempted failed; using result " +
                "from first attempt")
            print(attempt_1["text"], end="")

def try_url(url, fmt, clean=False):
    '''
    Return (Str, True) if succeeded; (Str, False) otherwise.
    '''
    result = {}
    analyte = analyze_url(url)
    if analyte:
        return {"text": analyte, "exit": True}
    try:
        user_agent = ("Mozilla/5.0 (X11; Linux i686; rv:31.0) Gecko/20100101 " +
            "Firefox/31.0 Iceweasel/31.5.0")
        headers = {"User-Agent": user_agent}
        response = requests.get(url, stream=True, headers=headers)
        url = response.url
        if  ("text/html" in response.headers["content-type"] or
                "application/pdf" in response.headers["content-type"]):
            logging.debug("HTML page or PDF file detected")
            # <title> is probably in the first around 10MB(FIXME:
            # 1000000 isn't 10MB)
            # FIXME: for html files, we can download less. the current
            # value is only intended for PDFs since those may require
            # more
            doc = response.iter_content(chunk_size=1000000)
            data = next(doc)
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"], data=data,
                    clean=clean),
                url,
                fmt
            )
        else:
            logging.debug("No HTML page detected")
            result["text"] = get_filetype_link(
                get_link_text(url, response.headers["content-type"]),
                url,
                fmt
            )
        result["exit"] = True
    except (requests.exceptions.MissingSchema,
            requests.exceptions.ConnectionError,
            requests.exceptions.InvalidSchema,
            urllib3.exceptions.LocationParseError):
        # since it's not a valid URL, just return it
        result["text"] = url
        result["exit"] = False
    return result

def analyze_url(url):
    """
    Look just at the URL to see if a suitable title text can be found.  This
    method is much faster than actually visiting the URL to find the title
    element in the downloaded file. We want to do this for special sites like
    Facebook, which doesn't allow anonymous downloading of certain pages, like
    group pages.

    Args:
        url: A string that is a URL

    Returns:
        A string that is the title text to be used. If no suitable title text
        can be produced, return the empty string, "".
    """
    try:
        tl = get_tld(url)
    except (tld.exceptions.TldBadUrl, tld.exceptions.TldDomainNotFound):
        logging.debug("bad TLD; trying with http:// prefix")
        try:
            tl = get_tld("http://" + url)
        except (tld.exceptions.TldBadUrl, tld.exceptions.TldDomainNotFound):
            logging.debug("still bad TLD; giving up")
            return ""
    if tl == "facebook.com" and "facebook.com/groups/" in url:
            return "Facebook group page post"
    return ""

def get_filetype_link(link_text, url, filetype):
    """
    Put the pieces together and produce a valid hyperlink for given output
    filetype.

    Args:
        link_text: A string representing the displayed or linked text when
            linking to something, e.g. "hello" in hello. This string should already be in
            the intended form; i.e. all HTML escapes should have been unescaped
            at this point.
        url: A string of the URL.
        filetype: A string of the output filetype. Accepted parameters are:
            "none", "html", "markdown", "latex".

    Returns:
        A string that is a valid hyperlink for the specified output filetype.
    """
    if filetype == "markdown":
        # From http://pandoc.org/README.html#backslash-escapes
        special_chars = "\\`*_{}[]()>#+-.!"
        result = ""
        for c in link_text:
            if c in special_chars:
                result += "\\" + c
            else:
                result += c
        return "[{link_text}]({url})".format(link_text=result, url=url)
    if filetype == "html":
        return '{link_text}'.format(url=url,
                link_text=html.escape(link_text))
    if filetype == "mediawiki":
        return "[{url} {link_text}]".format(url=url,
                link_text=link_text)
    if filetype == "latex":
        # LaTeX is really sensitive about special characters so this
        # probably needs a lot of tweaking
        special_chars = "$&%{_#"
        result = ""
        for c in link_text:
            if c in special_chars:
                result += "\\" + c
            elif c == "\\":
                result += "\\textbackslash{}"
            elif c == "~":
                result += "\\textasciitilde{}"
            else:
                result += c
        clean_url = ""
        for c in url:
            if c in special_chars or c in "~":
                clean_url += "\\" + c
            elif c == "\\":
                clean_url += "{\\textbackslash}"
            else:
                clean_url += c
        return ("\\href{%s}{%s}" % (clean_url, result))
    else:
        return "{link_text}: {url}".format(url=url, link_text=link_text)

def get_link_text(url, mime_type, data=None, clean=False):
    '''
    Take URL, MIME type, and optional data to produce the link text.
    '''
    tld = get_tld(url)
    result = "File on " + tld
    if mime_type.startswith("image"):
        result = "Image on " + tld
    elif  "application/pdf" in mime_type:
        logging.debug("PDF detected")
        # I need seek() for some reason so convert from bytes
        data = io.BytesIO(data)
        # fix this later, but I always get a "PdfReadWarning: Xref table
        # not zero-indexed" which should only happen when the -v flag is
        # present
        import warnings
        warnings.filterwarnings("ignore")
        pdf = PdfFileReader(data, strict=True)
        # PyPDF2 somehow thinks many PDFs are encrypted with the empty
        # string, so deal with that
        if pdf.isEncrypted:
            pdf.decrypt('')
        result = pdf.getDocumentInfo().title
        #result = "PDF on " + tld
    elif "text/html" in mime_type:
        try:
            soup = BeautifulSoup(data, 'html.parser')
            meta = soup.find_all("meta")
            og_title_lst = []
            twitter_title_lst = []
            meta_title_lst = []
            schema_lst = []
            for i in meta:
                if i.get("property") == "og:title":
                    og_title_lst.append(i.get("content"))
                elif i.get("property") == "twitter:title":
                    twitter_title_lst.append(i.get("content"))
                elif i.get("name") == "title":
                    meta_title_lst.append(i.get("content"))
                elif i.get("itemprop") == "name":
                    schema_lst.append(i.get("content"))
            if og_title_lst:
                logging.debug("found og:title")
                result = og_title_lst[0].strip()
            elif twitter_title_lst:
                logging.debug("found twitter title")
                result = twitter_title_lst[0].strip()
            elif meta_title_lst:
                logging.debug("found meta name title")
                result = meta_title_lst[0].strip()
            elif schema_lst:
                logging.debug("found schema title")
                result = schema_lst[0].strip()
            elif soup.title and soup.title.string:
                logging.debug("found title tag")
                result = html.unescape(soup.title.string)
                if clean:
                    result = messy_title_parse(result)
            else:
                logging.debug("no title found; using default")
                result = "Page on " + tld
        except AttributeError:
            # Probably just empty title when trying to get
            # soup.title.string
            logging.debug("FIXME: this isn't supposed to happen")
            result = "Page on " + tld
    if len(result) > 255:
        result = result[:253] + " …"

    return result

def messy_title_parse(title, url=None):
    # Even if nothing works, at least we'll have a whitespace-sanitized
    # title
    logging.debug("Cleaning messy title")
    result = title.strip()
    hyphen_split = result.split(" - ")
    bar_split = result.split(" | ")
    em_dash_split = result.split(" — ")
    colon_split = result.split(": ")
    if len(hyphen_split) > 1:
        # So there is actually more than one part, so we just take the
        # first and we're done.  This is for titles like "Post Title -
        # Site Name"
        result = hyphen_split[0]
    elif len(em_dash_split) > 1:
        result = em_dash_split[0]
    elif len(bar_split) > 1:
        result = bar_split[0]
    elif len(colon_split) > 1:
        # For titles like "Site Name: Post Title"
        result = colon_split[-1]
    return result

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from nassl.ssl_client import OpenSslVersionEnum, SslClient, OpenSslVerifyEnum
import socket

mozilla_store = 'tests/mozilla.pem'
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5)
sock.connect(('www.yahoo.com', 443))

ssl_client = SslClient(
    ssl_version=OpenSslVersionEnum.TLSV1_2,
    underlying_socket=sock,
    ssl_verify=OpenSslVerifyEnum.PEER,
    ssl_verify_locations=mozilla_store,
)
ssl_client.set_tlsext_status_ocsp()
ssl_client.do_handshake()

print('Received certificate chain')
for pem_cert in ssl_client.get_received_chain():
    print(pem_cert)

print('OCSP Stapling')
ocsp_resp = ssl_client.get_tlsext_status_ocsp_resp()
ocsp_resp.verify(mozilla_store)
print(ocsp_resp.as_dict())

print('\nCipher suite')
print(ssl_client.get_current_cipher_name())

print('\nHTTP response')
ssl_client.write(b'GET / HTTP/1.0\r\nUser-Agent: Test\r\nHost: www.google.com\r\n\r\n')
print(ssl_client.read(2048))
<commit_msg>Improve sample client<commit_after>from nassl.ssl_client import OpenSslVersionEnum, SslClient, OpenSslVerifyEnum
import socket

mozilla_store = 'tests/mozilla.pem'
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5)
sock.connect(('www.yahoo.com', 443))

ssl_client = SslClient(
    ssl_version=OpenSslVersionEnum.TLSV1_2,
    underlying_socket=sock,
    ssl_verify=OpenSslVerifyEnum.PEER,
    ssl_verify_locations=mozilla_store,
)
ssl_client.set_tlsext_status_ocsp()
ssl_client.do_handshake()

print('Received certificate chain')
for pem_cert in ssl_client.get_received_chain():
    print(pem_cert)

print('Verified certificate chain')
for pem_cert in ssl_client.get_verified_chain():
    print(pem_cert)

print('OCSP Stapling')
ocsp_resp = ssl_client.get_tlsext_status_ocsp_resp()
if ocsp_resp:
    ocsp_resp.verify(mozilla_store)
    print(ocsp_resp.as_dict())

print('\nCipher suite')
print(ssl_client.get_current_cipher_name())

print('\nHTTP response')
ssl_client.write(b'GET / HTTP/1.0\r\nUser-Agent: Test\r\nHost: www.google.com\r\n\r\n')
print(ssl_client.read(2048))
<|endoftext|>


<commit_before>#!/usr/bin/env python

###############################################################################
# Copyright 2017 The Apollo Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
"""SocketIO API."""
import google.protobuf.json_format as json_format

from config import Config
from modules.hmi.proto.socketio_api_pb2 import SocketIORequest
import hardware_api
import module_api
import ros_bridge_api
import tool_api

class SocketIOApi(object):
    """
    Socket IO API, which delegate request to target api, and broadcast result
    all HMI frontends.
    """
    API = {
        'hardware_api': hardware_api.HardwareApi,
        'module_api': module_api.ModuleApi,
        'ros_bridge_api': ros_bridge_api.RosBridgeApi,
        'tool_api': tool_api.ToolApi,
    }

    @classmethod
    def execute(cls, socketio_request_json):
        """Execute a command on the given API."""
        Config.log.info('SocketIOApi::execute %s', socketio_request_json)
        socketio_request = json_format.ParseDict(socketio_request_json,
                                                 SocketIORequest())

        api = cls.API.get(socketio_request.api_name)
        if api is None:
            Config.log.critical('Cannot find %s', socketio_request.api_name)
        cmd = eval('api.' + socketio_request.command_name)
        cmd(*socketio_request.args)
<commit_msg>HMI: Quick fix on miss-matched file name. (#663)<commit_after>#!/usr/bin/env python

###############################################################################
# Copyright 2017 The Apollo Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
"""SocketIO API."""
import google.protobuf.json_format as json_format

from config import Config
from modules.hmi.proto.hmi_message_pb2 import SocketIORequest
import hardware_api
import module_api
import ros_bridge_api
import tool_api

class SocketIOApi(object):
    """
    Socket IO API, which delegate request to target api, and broadcast result
    all HMI frontends.
    """
    API = {
        'hardware_api': hardware_api.HardwareApi,
        'module_api': module_api.ModuleApi,
        'ros_bridge_api': ros_bridge_api.RosBridgeApi,
        'tool_api': tool_api.ToolApi,
    }

    @classmethod
    def execute(cls, socketio_request_json):
        """Execute a command on the given API."""
        Config.log.info('SocketIOApi::execute %s', socketio_request_json)
        socketio_request = json_format.ParseDict(socketio_request_json,
                                                 SocketIORequest())

        api = cls.API.get(socketio_request.api_name)
        if api is None:
            Config.log.critical('Cannot find %s', socketio_request.api_name)
        cmd = eval('api.' + socketio_request.command_name)
        cmd(*socketio_request.args)
<|endoftext|>


<commit_before>from IPython.core.display import HTML
#from saspy import pysas34 as sas
import time
import logging
import os

# create logger
logger = logging.getLogger('')
logger.setLevel(logging.WARN)

class SAS_stat:
    def __init__(self, session, *args, **kwargs):
        '''Submit an initial set of macros to prepare the SAS system'''
        self.sas=session
        macro_path=os.path.dirname(os.path.realpath(__file__))
        code="options pagesize=max; %include '" + macro_path + '/' + "libname_gen.sas'; "
        self.sas._asubmit(code,"text")

        logger.debug("Initalization of SAS Macro: " + str(self.sas._getlog()))

    def _objectmethods(self,obj,*args):
        code  ="%listdata("
        code +=obj
        code +=");"
        logger.debug("Object Method macro call: " + str(code))
        res=self.sas.submit(code,"text")
        meth=res['LOG'].splitlines()
        logger.debug('SAS Log: ' + res['LOG'])
        objlist=meth[meth.index('startparse9878')+1:meth.index('endparse9878')]
        logger.debug("PROC attr list: " + str(objlist))
        return objlist

    def _makeProccallMacro(self,objtype,objname,kwargs):
        data=kwargs.get('data','')
        model=kwargs.get('model','')
        cls=kwargs.get('cls','')
        means=kwargs.get('means','')
        by =kwargs.get('by', '')
        est=kwargs.get('estimate','')
        weight=kwargs.get('weight','')
        lsmeans=kwargs.get('lsmeans','')

        code  = "%macro proccall(d);\n"
        if objtype == 'hpsplit':
           code += "proc %s data=%s.%s plots=all;\n" % (objtype, data.libref, data.table)
        else:
           code += "proc %s data=%s.%s plots(unpack)=all;\n" % (objtype, data.libref, data.table)
        #logger.debug("cls stuff: " +str(hasattr(self,'cls')) + ' ' + str(len(self.cls)))
        if len(cls):
            #logger.debug("cls stuff: " +str(hasattr(self,'cls')) + ' ' + str(len(self.cls)))
            code += "class %s;" % (cls)
        if len(model):
            code += "model %s;" % (model)
        if len(means):
            code += "means %s;" % (cls)
        code += "run; quit; %mend;\n"
        code += "%%mangobj(%s,%s,%s);" % (objname, objtype,data.table)
        logger.debug("Proc code submission: " + str(code))
        return (code)

    def hpsplit(self, **kwargs):
        '''Python method to call the HPSPLIT procedure\nDocumentation link: http://support.sas.com/documentation/cdl/en/stathpug/68163/HTML/default/viewer.htm#stathpug_hpsplit_overview.htm'''
        objtype='hpsplit'
        objname='hps'+self.sas._objcnt()  #translate to a libname so needs to be less than 8
        code=self._makeProccallMacro(objtype, objname, kwargs)
        logger.debug("HPSPLIT macro submission: " + str(code))
        self.sas._asubmit(code,"text")
        try:
            obj1=self._objectmethods(objname)
            logger.debug(obj1)
        except Exception:
            #print("Exception Block:", sys.exc_info()[0])
            obj1=[]

        return (SAS_results(obj1, self.sas, objname))

    def reg(self, **kwargs):
        objtype='reg'
        objname=objtype+self.sas._objcnt() #translate to a libname so needs to be less than 8
        code=self._makeProccallMacro(objtype, objname, kwargs)
        logger.debug("REG macro submission: " + str(code))
        self.sas._asubmit(code,"text")
        try:
            obj1=self._objectmethods(objname)
            logger.debug(obj1)
        except Exception:
            obj1=[]
        return (SAS_results(obj1, self.sas, objname))

    def mixed(self, **kwargs):
        objtype='mixed'
        objname='mix'+self.sas._objcnt()
        code=self._makeProccallMacro(objtype, objname, kwargs)
        logger.debug("Mixed Macro submission: " + str(code))
        self.sas._asubmit(code, "text")
        try:
            obj1=self._objectmethods(objname)
            logger.debug(obj1)
        except Exception:
            obj1=[]
        return (SAS_results(obj1, self.sas, objname))

    def glm(self, **kwargs):
        objtype='glm'
        objname=objtype+self.sas._objcnt() #translate to a libname so needs to be less than 8
        code=self._makeProccallMacro(objtype, objname, kwargs)
        logger.debug("GLM macro submission: " + str(code))
        self.sas._asubmit(code,"text")
        try:
            obj1=self._objectmethods(objname)
            logger.debug(obj1)
        except Exception:
            obj1=[]
        return (SAS_results(obj1, self.sas, objname))

from collections import namedtuple

class SAS_results(object):
    '''Return results from a SAS Model object'''
    def __init__(self,attrs, session, objname):

        self._attrs = attrs
        self._name = objname
        self.sas=session

    def __dir__(self):
        '''Overload dir method to return the attributes'''
        return self._attrs

    def __getattr__(self, attr):
        if attr.startswith('_'):
            return getattr(self, attr)
        if attr.upper() in self._attrs:
            #print(attr.upper())
            data = self._go_run_code(attr)
            '''
            if not attr.lower().endswith('plot'):
                libname, table = data.split()
                table_data = sasdata(libname, table)
                content = table_data.contents()
                # parse content
                headers = content[0]

                res = namedtuple('SAS Result', headers)
                results = [ res(x) for x in headers[1:] ]
            '''

        else:
             raise AttributeError
        return HTML(data)

    def _go_run_code(self, attr):
        #print(self._name, attr)
        code = '%%getdata(%s, %s);' % (self._name, attr)
        #print (code)
        res=self.sas.submit(code)
        return res['LST']

    def sasdata(self, table):
        x=self.sas.sasdata(table,'_'+self._name)
        return (x)

<commit_msg>add logistic method to sasstat<commit_after>from IPython.core.display import HTML
#from saspy import pysas34 as sas
import time
import logging
import os

# create logger
logger = logging.getLogger('')
logger.setLevel(logging.WARN)

class SAS_stat:
    def __init__(self, session, *args, **kwargs):
        '''Submit an initial set of macros to prepare the SAS system'''
        self.sas=session
        macro_path=os.path.dirname(os.path.realpath(__file__))
        code="options pagesize=max; %include '" + macro_path + '/' + "libname_gen.sas'; "
        self.sas._asubmit(code,"text")

        logger.debug("Initalization of SAS Macro: " + str(self.sas._getlog()))

    def _objectmethods(self,obj,*args):
        code  ="%listdata("
        code +=obj
        code +=");"
        logger.debug("Object Method macro call: " + str(code))
        res=self.sas.submit(code,"text")
        meth=res['LOG'].splitlines()
        logger.debug('SAS Log: ' + res['LOG'])
        objlist=meth[meth.index('startparse9878')+1:meth.index('endparse9878')]
        logger.debug("PROC attr list: " + str(objlist))
        return objlist

    def _makeProccallMacro(self,objtype,objname,kwargs):
        data=kwargs.get('data','')
        model=kwargs.get('model','')
        cls=kwargs.get('cls','')
        means=kwargs.get('means','')
        by =kwargs.get('by', '')
        est=kwargs.get('estimate','')
        weight=kwargs.get('weight','')
        lsmeans=kwargs.get('lsmeans','')

        code  = "%macro proccall(d);\n"
        if objtype == 'hpsplit':
           code += "proc %s data=%s.%s plots=all;\n" % (objtype, data.libref, data.table)
        else:
           code += "proc %s data=%s.%s plots(unpack)=all;\n" % (objtype, data.libref, data.table)
        #logger.debug("cls stuff: " +str(hasattr(self,'cls')) + ' ' + str(len(self.cls)))
        if len(cls):
            #logger.debug("cls stuff: " +str(hasattr(self,'cls')) + ' ' + str(len(self.cls)))
            code += "class %s;" % (cls)
        if len(model):
            code += "model %s;" % (model)
        if len(means):
            code += "means %s;" % (cls)
        code += "run; quit; %mend;\n"
        code += "%%mangobj(%s,%s,%s);" % (objname, objtype,data.table)
        logger.debug("Proc code submission: " + str(code))
        return (code)

    def hpsplit(self, **kwargs):
        '''Python method to call the HPSPLIT procedure\nDocumentation link: http://support.sas.com/documentation/cdl/en/stathpug/68163/HTML/default/viewer.htm#stathpug_hpsplit_overview.htm'''
        objtype='hpsplit'
        objname='hps'+self.sas._objcnt()  #translate to a libname so needs to be less than 8
        code=self._makeProccallMacro(objtype, objname, kwargs)
        logger.debug("HPSPLIT macro submission: " + str(code))
        self.sas._asubmit(code,"text")
        try:
            obj1=self._objectmethods(objname)
            logger.debug(obj1)
        except Exception:
            #print("Exception Block:", sys.exc_info()[0])
            obj1=[]

        return (SAS_results(obj1, self.sas, objname))

    def reg(self, **kwargs):
        objtype='reg'
        objname=objtype+self.sas._objcnt() #translate to a libname so needs to be less than 8
        code=self._makeProccallMacro(objtype, objname, kwargs)
        logger.debug("REG macro submission: " + str(code))
        self.sas._asubmit(code,"text")
        try:
            obj1=self._objectmethods(objname)
            logger.debug(obj1)
        except Exception:
            obj1=[]
        return (SAS_results(obj1, self.sas, objname))

    def mixed(self, **kwargs):
        objtype='mixed'
        objname='mix'+self.sas._objcnt()
        code=self._makeProccallMacro(objtype, objname, kwargs)
        logger.debug("Mixed Macro submission: " + str(code))
        self.sas._asubmit(code, "text")
        try:
            obj1=self._objectmethods(objname)
            logger.debug(obj1)
        except Exception:
            obj1=[]
        return (SAS_results(obj1, self.sas, objname))

    def glm(self, **kwargs):
        objtype='glm'
        objname=objtype+self.sas._objcnt() #translate to a libname so needs to be less than 8
        code=self._makeProccallMacro(objtype, objname, kwargs)
        logger.debug("GLM macro submission: " + str(code))
        self.sas._asubmit(code,"text")
        try:
            obj1=self._objectmethods(objname)
            logger.debug(obj1)
        except Exception:
            obj1=[]
        return (SAS_results(obj1, self.sas, objname))

    def logistic(self, **kwargs):
        objtype='logistic'
        objname='log'+self.sas._objcnt() #translate to a libname so needs to be less than 8
        code=self._makeProccallMacro(objtype, objname, kwargs)
        logger.debug("LOGISTIC macro submission: " + str(code))
        self.sas._asubmit(code,"text")
        try:
            obj1=self._objectmethods(objname)
            logger.debug(obj1)
        except Exception:
            obj1=[]
        return (SAS_results(obj1, self.sas, objname))

from collections import namedtuple

class SAS_results(object):
    '''Return results from a SAS Model object'''
    def __init__(self,attrs, session, objname):

        self._attrs = attrs
        self._name = objname
        self.sas=session

    def __dir__(self):
        '''Overload dir method to return the attributes'''
        return self._attrs

    def __getattr__(self, attr):
        if attr.startswith('_'):
            return getattr(self, attr)
        if attr.upper() in self._attrs:
            #print(attr.upper())
            data = self._go_run_code(attr)
            '''
            if not attr.lower().endswith('plot'):
                libname, table = data.split()
                table_data = sasdata(libname, table)
                content = table_data.contents()
                # parse content
                headers = content[0]

                res = namedtuple('SAS Result', headers)
                results = [ res(x) for x in headers[1:] ]
            '''

        else:
             raise AttributeError
        return HTML(data)

    def _go_run_code(self, attr):
        #print(self._name, attr)
        code = '%%getdata(%s, %s);' % (self._name, attr)
        #print (code)
        res=self.sas.submit(code)
        return res['LST']

    def sasdata(self, table):
        x=self.sas.sasdata(table,'_'+self._name)
        return (x)

<|endoftext|>


<commit_before># encoding: utf-8
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2008 PC Solutions (<http://pcsol.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time

from openerp.osv import fields, osv
from openerp.tools.translate import _
import openerp.addons.decimal_precision as dp

class account_cashbox_line(osv.osv):

    """ Cash Box Details """

    _name = 'account.cashbox.line'
    _description = 'CashBox Line'
    _rec_name = 'pieces'

    def _sub_total(self, cr, uid, ids, name, arg, context=None):

        """ Calculates Sub total
        @param name: Names of fields.
        @param arg: User defined arguments
        @return: Dictionary of values.
        """
        res = {}
        for obj in self.browse(cr, uid, ids, context=context):
            res[obj.id] = {
                'subtotal_opening' : obj.pieces * obj.number_opening,
                'subtotal_closing' : obj.pieces * obj.number_closing,
            }
        return res

    def on_change_sub_opening(self, cr, uid, ids, pieces, number, *a):
        """ Compute the subtotal for the opening """
        return {'value' : {'subtotal_opening' : (pieces * number) or 0.0 }}

    def on_change_sub_closing(self, cr, uid, ids, pieces, number, *a):
        """ Compute the subtotal for the closing """
        return {'value' : {'subtotal_closing' : (pieces * number) or 0.0 }}

    _columns = {
        'pieces': fields.float('Unit of Currency', digits_compute=dp.get_precision('Account')),
        'number_opening' : fields.integer('Number of Units', help='Opening Unit Numbers'),
        'number_closing' : fields.integer('Number of Units', help='Closing Unit Numbers'),
        'subtotal_opening': fields.function(_sub_total, string='Opening Subtotal', type='float', digits_compute=dp.get_precision('Account'), multi='subtotal'),
        'subtotal_closing': fields.function(_sub_total, string='Closing Subtotal', type='float', digits_compute=dp.get_precision('Account'), multi='subtotal'),
        'bank_statement_id' : fields.many2one('account.bank.statement', ondelete='cascade'),
     }

account_cashbox_line()

class account_cash_statement(osv.osv):

    _inherit = 'account.bank.statement'

    def _update_balances(self, cr, uid, ids, context=None):
        """
            Set starting and ending balances according to pieces count
        """
        res = {}
        for statement in self.browse(cr, uid, ids, context=context):
            if (statement.journal_id.type not in ('cash',)) or (not statement.journal_id.cash_control):
                continue
            start = end = 0
            for line in statement.details_ids:
                start += line.subtotal_opening
                end += line.subtotal_closing
            data = {
                'balance_start': start,
                'balance_end_real': end,
            }
            res[statement.id] = data
            super(account_cash_statement, self).write(cr, uid, [statement.id], data, context=context)
        return res

    def _get_sum_entry_encoding(self, cr, uid, ids, name, arg, context=None):

        """ Find encoding total of statements "
        @param name: Names of fields.
        @param arg: User defined arguments
        @return: Dictionary of values.
        """
        res = {}
        for statement in self.browse(cr, uid, ids, context=context):
            res[statement.id] = sum((line.amount for line in statement.line_ids), 0.0)
        return res

    def _get_company(self, cr, uid, context=None):
        user_pool = self.pool.get('res.users')
        company_pool = self.pool.get('res.company')
        user = user_pool.browse(cr, uid, uid, context=context)
        company_id = user.company_id
        if not company_id:
            company_id = company_pool.search(cr, uid, [])
        return company_id and company_id[0] or False

    def _get_statement_from_line(self, cr, uid, ids, context=None):
        result = {}
        for line in self.pool.get('account.bank.statement.line').browse(cr, uid, ids, context=context):
            result[line.statement_id.id] = True
        return result.keys()

    def _compute_difference(self, cr, uid, ids, fieldnames, args, context=None):
        result =  dict.fromkeys(ids, 0.0)

        for obj in self.browse(cr, uid, ids, context=context):
            result[obj.id] = obj.balance_end_real - obj.balance_end

        return result

    def _compute_last_closing_balance(self, cr, uid, ids, fieldnames, args, context=None):
        result = dict.fromkeys(ids, 0.0)

        for obj in self.browse(cr, uid, ids, context=context):
            if obj.state == 'draft':
                statement_ids = self.search(cr, uid,
                    [('journal_id', '=', obj.journal_id.id),('state', '=', 'confirm')],
                    order='create_date desc',
                    limit=1,
                    context=context
                )

                if not statement_ids:
                    continue
                else:
                    st = self.browse(cr, uid, statement_ids[0], context=context)
                    result[obj.id] = st.balance_end_real

        return result

    def onchange_journal_id(self, cr, uid, ids, journal_id, context=None):
        result = super(account_cash_statement, self).onchange_journal_id(cr, uid, ids, journal_id)

        if not journal_id:
            return result

        statement_ids = self.search(cr, uid,
                [('journal_id', '=', journal_id),('state', '=', 'confirm')],
                order='create_date desc',
                limit=1,
                context=context
        )

        if not statement_ids:
            return result

        st = self.browse(cr, uid, statement_ids[0], context=context)
        result.setdefault('value', {}).update({'last_closing_balance' : st.balance_end_real})

        return result

    _columns = {
        'total_entry_encoding': fields.function(_get_sum_entry_encoding, string="Total Transactions",
            store = {
                'account.bank.statement': (lambda self, cr, uid, ids, context=None: ids, ['line_ids','move_line_ids'], 10),
                'account.bank.statement.line': (_get_statement_from_line, ['amount'], 10),
            }),
        'closing_date': fields.datetime("Closed On"),
        'details_ids' : fields.one2many('account.cashbox.line', 'bank_statement_id', string='CashBox Lines'),
        'opening_details_ids' : fields.one2many('account.cashbox.line', 'bank_statement_id', string='Opening Cashbox Lines'),
        'closing_details_ids' : fields.one2many('account.cashbox.line', 'bank_statement_id', string='Closing Cashbox Lines'),
        'user_id': fields.many2one('res.users', 'Responsible', required=False),
        'difference' : fields.function(_compute_difference, method=True, string="Difference", type="float"),
        'last_closing_balance' : fields.function(_compute_last_closing_balance, method=True, string='Last Closing Balance', type='float'),
    }
    _defaults = {
        'state': 'draft',
        'date': lambda self, cr, uid, context={}: context.get('date', time.strftime("%Y-%m-%d %H:%M:%S")),
        'user_id': lambda self, cr, uid, context=None: uid,
    }

    def create(self, cr, uid, vals, context=None):
        journal = False
        if vals.get('journal_id'):
            journal = self.pool.get('account.journal').browse(cr, uid, vals['journal_id'], context=context)
        if journal and (journal.type == 'cash') and not vals.get('details_ids'):
            vals['details_ids'] = []

            last_pieces = None

            if journal.with_last_closing_balance == True:
                domain = [('journal_id', '=', journal.id),
                          ('state', '=', 'confirm')]
                last_bank_statement_ids = self.search(cr, uid, domain, limit=1, order='create_date desc', context=context)
                if last_bank_statement_ids:
                    last_bank_statement = self.browse(cr, uid, last_bank_statement_ids[0], context=context)

                    last_pieces = dict(
                        (line.pieces, line.number_closing) for line in last_bank_statement.details_ids
                    )

            for value in journal.cashbox_line_ids:
                nested_values = {
                    'number_closing' : 0,
                    'number_opening' : last_pieces.get(value.pieces, 0) if isinstance(last_pieces, dict) else 0,
                    'pieces' : value.pieces
                }

                vals['details_ids'].append([0, False, nested_values])

        res_id = super(account_cash_statement, self).create(cr, uid, vals, context=context)
        self._update_balances(cr, uid, [res_id], context)
        return res_id

    def write(self, cr, uid, ids, vals, context=None):
        """
        Update redord(s) comes in {ids}, with new value comes as {vals}
        return True on success, False otherwise

        @param cr: cursor to database
        @param user: id of current user
        @param ids: list of record ids to be update
        @param vals: dict of new values to be set
        @param context: context arguments, like lang, time zone

        @return: True on success, False otherwise
        """

        res = super(account_cash_statement, self).write(cr, uid, ids, vals, context=context)
        self._update_balances(cr, uid, ids, context)
        return res

    def _user_allow(self, cr, uid, statement_id, context=None):
        return True

    def button_open(self, cr, uid, ids, context=None):
        """ Changes statement state to Running.
        @return: True
        """
        obj_seq = self.pool.get('ir.sequence')
        if context is None:
            context = {}
        statement_pool = self.pool.get('account.bank.statement')
        for statement in statement_pool.browse(cr, uid, ids, context=context):
            vals = {}
            if not self._user_allow(cr, uid, statement.id, context=context):
                raise osv.except_osv(_('Error!'), (_('You do not have rights to open this %s journal !') % (statement.journal_id.name, )))

            if statement.name and statement.name == '/':
                c = {'fiscalyear_id': statement.period_id.fiscalyear_id.id}
                if statement.journal_id.sequence_id:
                    st_number = obj_seq.next_by_id(cr, uid, statement.journal_id.sequence_id.id, context=c)
                else:
                    st_number = obj_seq.next_by_code(cr, uid, 'account.cash.statement', context=c)
                vals.update({
                    'name': st_number
                })

            vals.update({
                'state': 'open',
            })
            self.write(cr, uid, [statement.id], vals, context=context)
        return True

    def statement_close(self, cr, uid, ids, journal_type='bank', context=None):
        if journal_type == 'bank':
            return super(account_cash_statement, self).statement_close(cr, uid, ids, journal_type, context)
        vals = {
            'state':'confirm',
            'closing_date': time.strftime("%Y-%m-%d %H:%M:%S")
        }
        return self.write(cr, uid, ids, vals, context=context)

    def check_status_condition(self, cr, uid, state, journal_type='bank'):
        if journal_type == 'bank':
            return super(account_cash_statement, self).check_status_condition(cr, uid, state, journal_type)
        return state=='open'

    def button_confirm_cash(self, cr, uid, ids, context=None):
        super(account_cash_statement, self).button_confirm_bank(cr, uid, ids, context=context)
        absl_proxy = self.pool.get('account.bank.statement.line')

        TABLES = ((_('Profit'), 'profit_account_id'), (_('Loss'), 'loss_account_id'),)

        for obj in self.browse(cr, uid, ids, context=context):
            if obj.difference == 0.0:
                continue

            for item_label, item_account in TABLES:
                if getattr(obj.journal_id, item_account):
                    raise osv.except_osv(_('Error!'),
                                         _('There is no %s Account on the journal %s.') % (item_label, obj.journal_id.name,))

            is_profit = obj.difference < 0.0

            account = getattr(obj.journal_id, TABLES[is_profit][1])

            values = {
                'statement_id' : obj.id,
                'journal_id' : obj.journal_id.id,
                'account_id' : account.id,
                'amount' : obj.difference,
                'name' : 'Exceptional %s' % TABLES[is_profit][0],
            }

            absl_proxy.create(cr, uid, values, context=context)

        return self.write(cr, uid, ids, {'closing_date': time.strftime("%Y-%m-%d %H:%M:%S")}, context=context)

account_cash_statement()

class account_journal(osv.osv):
    _inherit = 'account.journal'

    def _default_cashbox_line_ids(self, cr, uid, context=None):
        # Return a list of coins in Euros.
        result = [
            dict(pieces=value) for value in [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500]
        ]
        return result

    _columns = {
        'cashbox_line_ids' : fields.one2many('account.journal.cashbox.line', 'journal_id', 'CashBox'),
    }

    _defaults = {
        'cashbox_line_ids' : _default_cashbox_line_ids,
    }

account_journal()

class account_journal_cashbox_line(osv.osv):
    _name = 'account.journal.cashbox.line'
    _rec_name = 'pieces'
    _columns = {
        'pieces': fields.float('Values', digits_compute=dp.get_precision('Account')),
        'journal_id' : fields.many2one('account.journal', 'Journal', required=True, select=1),
    }

    _order = 'pieces asc'

account_journal_cashbox_line()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[FIX] lp:1102612. Cannot delete a journal.<commit_after># encoding: utf-8
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2008 PC Solutions (<http://pcsol.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time

from openerp.osv import fields, osv
from openerp.tools.translate import _
import openerp.addons.decimal_precision as dp

class account_cashbox_line(osv.osv):

    """ Cash Box Details """

    _name = 'account.cashbox.line'
    _description = 'CashBox Line'
    _rec_name = 'pieces'

    def _sub_total(self, cr, uid, ids, name, arg, context=None):

        """ Calculates Sub total
        @param name: Names of fields.
        @param arg: User defined arguments
        @return: Dictionary of values.
        """
        res = {}
        for obj in self.browse(cr, uid, ids, context=context):
            res[obj.id] = {
                'subtotal_opening' : obj.pieces * obj.number_opening,
                'subtotal_closing' : obj.pieces * obj.number_closing,
            }
        return res

    def on_change_sub_opening(self, cr, uid, ids, pieces, number, *a):
        """ Compute the subtotal for the opening """
        return {'value' : {'subtotal_opening' : (pieces * number) or 0.0 }}

    def on_change_sub_closing(self, cr, uid, ids, pieces, number, *a):
        """ Compute the subtotal for the closing """
        return {'value' : {'subtotal_closing' : (pieces * number) or 0.0 }}

    _columns = {
        'pieces': fields.float('Unit of Currency', digits_compute=dp.get_precision('Account')),
        'number_opening' : fields.integer('Number of Units', help='Opening Unit Numbers'),
        'number_closing' : fields.integer('Number of Units', help='Closing Unit Numbers'),
        'subtotal_opening': fields.function(_sub_total, string='Opening Subtotal', type='float', digits_compute=dp.get_precision('Account'), multi='subtotal'),
        'subtotal_closing': fields.function(_sub_total, string='Closing Subtotal', type='float', digits_compute=dp.get_precision('Account'), multi='subtotal'),
        'bank_statement_id' : fields.many2one('account.bank.statement', ondelete='cascade'),
     }

account_cashbox_line()

class account_cash_statement(osv.osv):

    _inherit = 'account.bank.statement'

    def _update_balances(self, cr, uid, ids, context=None):
        """
            Set starting and ending balances according to pieces count
        """
        res = {}
        for statement in self.browse(cr, uid, ids, context=context):
            if (statement.journal_id.type not in ('cash',)) or (not statement.journal_id.cash_control):
                continue
            start = end = 0
            for line in statement.details_ids:
                start += line.subtotal_opening
                end += line.subtotal_closing
            data = {
                'balance_start': start,
                'balance_end_real': end,
            }
            res[statement.id] = data
            super(account_cash_statement, self).write(cr, uid, [statement.id], data, context=context)
        return res

    def _get_sum_entry_encoding(self, cr, uid, ids, name, arg, context=None):

        """ Find encoding total of statements "
        @param name: Names of fields.
        @param arg: User defined arguments
        @return: Dictionary of values.
        """
        res = {}
        for statement in self.browse(cr, uid, ids, context=context):
            res[statement.id] = sum((line.amount for line in statement.line_ids), 0.0)
        return res

    def _get_company(self, cr, uid, context=None):
        user_pool = self.pool.get('res.users')
        company_pool = self.pool.get('res.company')
        user = user_pool.browse(cr, uid, uid, context=context)
        company_id = user.company_id
        if not company_id:
            company_id = company_pool.search(cr, uid, [])
        return company_id and company_id[0] or False

    def _get_statement_from_line(self, cr, uid, ids, context=None):
        result = {}
        for line in self.pool.get('account.bank.statement.line').browse(cr, uid, ids, context=context):
            result[line.statement_id.id] = True
        return result.keys()

    def _compute_difference(self, cr, uid, ids, fieldnames, args, context=None):
        result =  dict.fromkeys(ids, 0.0)

        for obj in self.browse(cr, uid, ids, context=context):
            result[obj.id] = obj.balance_end_real - obj.balance_end

        return result

    def _compute_last_closing_balance(self, cr, uid, ids, fieldnames, args, context=None):
        result = dict.fromkeys(ids, 0.0)

        for obj in self.browse(cr, uid, ids, context=context):
            if obj.state == 'draft':
                statement_ids = self.search(cr, uid,
                    [('journal_id', '=', obj.journal_id.id),('state', '=', 'confirm')],
                    order='create_date desc',
                    limit=1,
                    context=context
                )

                if not statement_ids:
                    continue
                else:
                    st = self.browse(cr, uid, statement_ids[0], context=context)
                    result[obj.id] = st.balance_end_real

        return result

    def onchange_journal_id(self, cr, uid, ids, journal_id, context=None):
        result = super(account_cash_statement, self).onchange_journal_id(cr, uid, ids, journal_id)

        if not journal_id:
            return result

        statement_ids = self.search(cr, uid,
                [('journal_id', '=', journal_id),('state', '=', 'confirm')],
                order='create_date desc',
                limit=1,
                context=context
        )

        if not statement_ids:
            return result

        st = self.browse(cr, uid, statement_ids[0], context=context)
        result.setdefault('value', {}).update({'last_closing_balance' : st.balance_end_real})

        return result

    _columns = {
        'total_entry_encoding': fields.function(_get_sum_entry_encoding, string="Total Transactions",
            store = {
                'account.bank.statement': (lambda self, cr, uid, ids, context=None: ids, ['line_ids','move_line_ids'], 10),
                'account.bank.statement.line': (_get_statement_from_line, ['amount'], 10),
            }),
        'closing_date': fields.datetime("Closed On"),
        'details_ids' : fields.one2many('account.cashbox.line', 'bank_statement_id', string='CashBox Lines'),
        'opening_details_ids' : fields.one2many('account.cashbox.line', 'bank_statement_id', string='Opening Cashbox Lines'),
        'closing_details_ids' : fields.one2many('account.cashbox.line', 'bank_statement_id', string='Closing Cashbox Lines'),
        'user_id': fields.many2one('res.users', 'Responsible', required=False),
        'difference' : fields.function(_compute_difference, method=True, string="Difference", type="float"),
        'last_closing_balance' : fields.function(_compute_last_closing_balance, method=True, string='Last Closing Balance', type='float'),
    }
    _defaults = {
        'state': 'draft',
        'date': lambda self, cr, uid, context={}: context.get('date', time.strftime("%Y-%m-%d %H:%M:%S")),
        'user_id': lambda self, cr, uid, context=None: uid,
    }

    def create(self, cr, uid, vals, context=None):
        journal = False
        if vals.get('journal_id'):
            journal = self.pool.get('account.journal').browse(cr, uid, vals['journal_id'], context=context)
        if journal and (journal.type == 'cash') and not vals.get('details_ids'):
            vals['details_ids'] = []

            last_pieces = None

            if journal.with_last_closing_balance == True:
                domain = [('journal_id', '=', journal.id),
                          ('state', '=', 'confirm')]
                last_bank_statement_ids = self.search(cr, uid, domain, limit=1, order='create_date desc', context=context)
                if last_bank_statement_ids:
                    last_bank_statement = self.browse(cr, uid, last_bank_statement_ids[0], context=context)

                    last_pieces = dict(
                        (line.pieces, line.number_closing) for line in last_bank_statement.details_ids
                    )

            for value in journal.cashbox_line_ids:
                nested_values = {
                    'number_closing' : 0,
                    'number_opening' : last_pieces.get(value.pieces, 0) if isinstance(last_pieces, dict) else 0,
                    'pieces' : value.pieces
                }

                vals['details_ids'].append([0, False, nested_values])

        res_id = super(account_cash_statement, self).create(cr, uid, vals, context=context)
        self._update_balances(cr, uid, [res_id], context)
        return res_id

    def write(self, cr, uid, ids, vals, context=None):
        """
        Update redord(s) comes in {ids}, with new value comes as {vals}
        return True on success, False otherwise

        @param cr: cursor to database
        @param user: id of current user
        @param ids: list of record ids to be update
        @param vals: dict of new values to be set
        @param context: context arguments, like lang, time zone

        @return: True on success, False otherwise
        """

        res = super(account_cash_statement, self).write(cr, uid, ids, vals, context=context)
        self._update_balances(cr, uid, ids, context)
        return res

    def _user_allow(self, cr, uid, statement_id, context=None):
        return True

    def button_open(self, cr, uid, ids, context=None):
        """ Changes statement state to Running.
        @return: True
        """
        obj_seq = self.pool.get('ir.sequence')
        if context is None:
            context = {}
        statement_pool = self.pool.get('account.bank.statement')
        for statement in statement_pool.browse(cr, uid, ids, context=context):
            vals = {}
            if not self._user_allow(cr, uid, statement.id, context=context):
                raise osv.except_osv(_('Error!'), (_('You do not have rights to open this %s journal !') % (statement.journal_id.name, )))

            if statement.name and statement.name == '/':
                c = {'fiscalyear_id': statement.period_id.fiscalyear_id.id}
                if statement.journal_id.sequence_id:
                    st_number = obj_seq.next_by_id(cr, uid, statement.journal_id.sequence_id.id, context=c)
                else:
                    st_number = obj_seq.next_by_code(cr, uid, 'account.cash.statement', context=c)
                vals.update({
                    'name': st_number
                })

            vals.update({
                'state': 'open',
            })
            self.write(cr, uid, [statement.id], vals, context=context)
        return True

    def statement_close(self, cr, uid, ids, journal_type='bank', context=None):
        if journal_type == 'bank':
            return super(account_cash_statement, self).statement_close(cr, uid, ids, journal_type, context)
        vals = {
            'state':'confirm',
            'closing_date': time.strftime("%Y-%m-%d %H:%M:%S")
        }
        return self.write(cr, uid, ids, vals, context=context)

    def check_status_condition(self, cr, uid, state, journal_type='bank'):
        if journal_type == 'bank':
            return super(account_cash_statement, self).check_status_condition(cr, uid, state, journal_type)
        return state=='open'

    def button_confirm_cash(self, cr, uid, ids, context=None):
        super(account_cash_statement, self).button_confirm_bank(cr, uid, ids, context=context)
        absl_proxy = self.pool.get('account.bank.statement.line')

        TABLES = ((_('Profit'), 'profit_account_id'), (_('Loss'), 'loss_account_id'),)

        for obj in self.browse(cr, uid, ids, context=context):
            if obj.difference == 0.0:
                continue

            for item_label, item_account in TABLES:
                if getattr(obj.journal_id, item_account):
                    raise osv.except_osv(_('Error!'),
                                         _('There is no %s Account on the journal %s.') % (item_label, obj.journal_id.name,))

            is_profit = obj.difference < 0.0

            account = getattr(obj.journal_id, TABLES[is_profit][1])

            values = {
                'statement_id' : obj.id,
                'journal_id' : obj.journal_id.id,
                'account_id' : account.id,
                'amount' : obj.difference,
                'name' : 'Exceptional %s' % TABLES[is_profit][0],
            }

            absl_proxy.create(cr, uid, values, context=context)

        return self.write(cr, uid, ids, {'closing_date': time.strftime("%Y-%m-%d %H:%M:%S")}, context=context)

account_cash_statement()

class account_journal(osv.osv):
    _inherit = 'account.journal'

    def _default_cashbox_line_ids(self, cr, uid, context=None):
        # Return a list of coins in Euros.
        result = [
            dict(pieces=value) for value in [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500]
        ]
        return result

    _columns = {
        'cashbox_line_ids' : fields.one2many('account.journal.cashbox.line', 'journal_id', 'CashBox'),
    }

    _defaults = {
        'cashbox_line_ids' : _default_cashbox_line_ids,
    }

account_journal()

class account_journal_cashbox_line(osv.osv):
    _name = 'account.journal.cashbox.line'
    _rec_name = 'pieces'
    _columns = {
        'pieces': fields.float('Values', digits_compute=dp.get_precision('Account')),
        'journal_id' : fields.many2one('account.journal', 'Journal', required=True, select=1, ondelete="cascade"),
    }

    _order = 'pieces asc'

account_journal_cashbox_line()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>from IPython.core.display import HTML
from saspy import pysas34 as sas
import time
import logging

# create logger
logger = logging.getLogger('')
logger.setLevel(logging.WARN)

class sasstat:
    def __init__(self, *args, **kwargs):
        '''Submit an initial set of macros to prepare the SAS system'''
        code="options pagesize=max; %include '/root/jared/metis/saspy_pip/saspy/libname_gen.sas'; "
        sas._submit(code,"text")
        logger.debug("Initalization of SAS Macro: " + str(sas._getlog()))

    def __flushlst__(self):
        lst = b'hi'
        while(len(lst) > 0):
           lst = saspid.stdout.read1(4096)
           continue

    def _objectmethods(self,obj,*args):
        self.obj=obj
        clear=sas._getlog(1)
        code  ="%listdata("
        code +=self.obj
        code +=");"
        logger.debug("Object Method macro call: " + str(code))
        sas._submit(code,"text")
        meth=sas._getlog().splitlines()
        logger.debug('SAS Log: ' + str(meth))
        objlist=meth[meth.index('startparse9878')+1:meth.index('endparse9878')]
        logger.debug("PROC attr list: " + str(objlist))
        return objlist

    def _makeProccallMacro(self):
        code  = "%macro proccall(d);\n"
        code += "proc %s data=%s.%s plots(unpack)=all;\n" % (self.objtype, self.data.libref, self.data.table)
        if len(self.model):
            code += "model %s;" % (self.model)
        if hasattr(self, 'cls') and len(self.cls):
            code += "class %s;" % (self.cls)
        if hasattr(self, 'means') and len(self.means):
            code += "means %s;" % (self.cls)
        code += "run; quit; %mend;\n"
        code += "%%mangobj(%s,%s,%s);" % (self.objname, self.objtype,self.data.table)
        logger.debug("Proc code submission: " + str(code))
        return (code)

    def hpsplit(self, model='', data=None, **kwargs):
        self.model=model
        self.data=data
        self.objtype='hps'
        self.objname='hps1' #how to give this a better name -- translate to a libname so needs to be less than 8
        code=self._makeProccallMacro()
        logger.debug("HPSPLIT macro submission: " + str(code))
        sas._submit(code,"text")
        try:
            obj1=self._objectmethods(self.objname)
            logger.debug(obj1)
        except Exception:
            #print("Exception Block:", sys.exc_info()[0])
            obj1=[]

        return (Results(obj1,self.objname))

    def reg(self, model='', data=None, **kwargs):
        self.model=model
        self.data=data
        self.objtype='reg'
        self.objname='reg1' #how to give this a better name
        code=self._makeProccallMacro()
        logger.debug("REG macro submission: " + str(code))
        sas._submit(code,"text")
        try:
            obj1=self._objectmethods(self.objname)
            logger.debug(obj1)
        except Exception:
            obj1=[]
        return (Results(obj1,self.objname))

    def glm(self, model='', data=None, **kwargs):
        self.model=model
        self.data=data
        self.cls=kwargs.get('cls', '')
        self.by =kwargs.get('by', '')
        self.est=kwargs.get('estimate','')
        self.weight=kwargs.get('weight','')
        self.lsmeans=kwargs.get('lsmeans','')
        self.means=kwargs.get('means','')
        self.objtype='glm'
        self.objname='glm1' #how to give this a better name
        code=self._makeProccallMacro()
        logger.debug("GLM macro submission: " + str(code))
        sas._submit(code,"text")
        try:
            obj1=self._objectmethods(self.objname)
            logger.debug(obj1)
        except Exception:
            obj1=[]
        return (Results(obj1,self.objname))

from collections import namedtuple

class Results(object):
    '''Return results from a SAS Model object'''
    def __init__(self, attrs, objname):

        self._attrs = attrs
        self._name = objname
        logger.debug("attributes passed to Results: "+ self._attrs,self._name)

    def __dir__(self):
        '''Overload dir method to return the attributes'''
        return self._attrs

    def __getattr__(self, attr):
        if attr.startswith('_'):
            return getattr(self, attr)
        if attr.upper() in self._attrs:
            #print(attr.upper())
            data = self._go_run_code(attr)
            '''
            if not attr.lower().endswith('plot'):
                libname, table = data.split()
                table_data = sasdata(libname, table)
                content = table_data.contents()
                # parse content
                headers = content[0]

                res = namedtuple('SAS Result', headers)
                results = [ res(x) for x in headers[1:] ]
            '''

        else:
             raise AttributeError
        return HTML(data)

    def _go_run_code(self, attr):
        #print(self._name, attr)
        code = '%%getdata(%s, %s);' % (self._name, attr)
        #print (code)
        sas._submit(code)
        return sas._getlst()

<commit_msg>updates to sasstat.py\n this update allows optional parameters to be passed to the model methods<commit_after>from IPython.core.display import HTML
from saspy import pysas34 as sas
import time
import logging

# create logger
logger = logging.getLogger('')
logger.setLevel(logging.WARN)

class sasstat:
    def __init__(self, *args, **kwargs):
        '''Submit an initial set of macros to prepare the SAS system'''
        code="options pagesize=max; %include '/root/jared/metis/saspy_pip/saspy/libname_gen.sas'; "
        sas._submit(code,"text")
        logger.debug("Initalization of SAS Macro: " + str(sas._getlog()))

    def __flushlst__(self):
        lst = b'hi'
        while(len(lst) > 0):
           lst = saspid.stdout.read1(4096)
           continue

    def _objectmethods(self,obj,*args):
        self.obj=obj
        clear=sas._getlog(1)
        code  ="%listdata("
        code +=self.obj
        code +=");"
        logger.debug("Object Method macro call: " + str(code))
        sas._submit(code,"text")
        meth=sas._getlog().splitlines()
        logger.debug('SAS Log: ' + str(meth))
        objlist=meth[meth.index('startparse9878')+1:meth.index('endparse9878')]
        logger.debug("PROC attr list: " + str(objlist))
        return objlist

    def _makeProccallMacro(self):
        code  = "%macro proccall(d);\n"
        code += "proc %s data=%s.%s plots(unpack)=all;\n" % (self.objtype, self.data.libref, self.data.table)
        if len(self.model):
            code += "model %s;" % (self.model)
        if hasattr(self, 'cls') and len(self.cls):
            code += "class %s;" % (self.cls)
        if hasattr(self, 'means') and len(self.means):
            code += "means %s;" % (self.cls)
        code += "run; quit; %mend;\n"
        code += "%%mangobj(%s,%s,%s);" % (self.objname, self.objtype,self.data.table)
        logger.debug("Proc code submission: " + str(code))
        return (code)

    def hpsplit(self, model='', data=None, **kwargs):
        self.model=model
        self.data=data
        self.objtype='hps'
        self.objname='hps1' #how to give this a better name -- translate to a libname so needs to be less than 8
        code=self._makeProccallMacro()
        logger.debug("HPSPLIT macro submission: " + str(code))
        sas._submit(code,"text")
        try:
            obj1=self._objectmethods(self.objname)
            logger.debug(obj1)
        except Exception:
            #print("Exception Block:", sys.exc_info()[0])
            obj1=[]

        return (Results(obj1,self.objname))

    def reg(self, model='', data=None, **kwargs):
        self.model=model
        self.data=data
        self.objtype='reg'
        self.objname='reg1' #how to give this a better name
        code=self._makeProccallMacro()
        logger.debug("REG macro submission: " + str(code))
        sas._submit(code,"text")
        try:
            obj1=self._objectmethods(self.objname)
            logger.debug(obj1)
        except Exception:
            obj1=[]
        return (Results(obj1,self.objname))
    #def glm(self, model='', data=None, **kwargs):
    def glm(self, **kwargs):
        self.model=kwargs.get('model','')
        self.data=kwargs.get('data','')
        self.cls=kwargs.get('cls', '')
        self.by =kwargs.get('by', '')
        self.est=kwargs.get('estimate','')
        self.weight=kwargs.get('weight','')
        self.lsmeans=kwargs.get('lsmeans','')
        self.means=kwargs.get('means','')
        self.objtype='glm'
        self.objname='glm1' #how to give this a better name
        code=self._makeProccallMacro()
        logger.debug("GLM macro submission: " + str(code))
        sas._submit(code,"text")
        try:
            obj1=self._objectmethods(self.objname)
            logger.debug(obj1)
        except Exception:
            obj1=[]
        return (Results(obj1,self.objname))

from collections import namedtuple

class Results(object):
    '''Return results from a SAS Model object'''
    def __init__(self, attrs, objname):

        self._attrs = attrs
        self._name = objname

    def __dir__(self):
        '''Overload dir method to return the attributes'''
        return self._attrs

    def __getattr__(self, attr):
        if attr.startswith('_'):
            return getattr(self, attr)
        if attr.upper() in self._attrs:
            #print(attr.upper())
            data = self._go_run_code(attr)
            '''
            if not attr.lower().endswith('plot'):
                libname, table = data.split()
                table_data = sasdata(libname, table)
                content = table_data.contents()
                # parse content
                headers = content[0]

                res = namedtuple('SAS Result', headers)
                results = [ res(x) for x in headers[1:] ]
            '''

        else:
             raise AttributeError
        return HTML(data)

    def _go_run_code(self, attr):
        #print(self._name, attr)
        code = '%%getdata(%s, %s);' % (self._name, attr)
        #print (code)
        sas._submit(code)
        return sas._getlst()

<|endoftext|>


<commit_before>import logging
import re
import string
from collections import OrderedDict

from ..constructs import *

class BaseAnalyzer(object):
    '''
    The analyzers are responsible for taking raw instructions from the executable and transforming them
    into higher-level constructs. This includes identifying functions, basic blocks, etc.

    The analyzers also provide some helper methods (ins_*) which are quick ways to determine what an instruction does.
    This can include determining if a instruction is sensitive to location, is a call/jump, etc.
    '''
    def __init__(self, executable):
        self.executable = executable

        # Dictionary of vaddr: instruction for quick lookups.
        # We use an OrderedDict so we can just `for addr in ins_map` and not have to worry about sorting
        self.ins_map = OrderedDict()

    def __repr__(self):
        return '<{} for {} {} \'{}\'>'.format(self.__class__.__name__,
                                              self.executable.architecture,
                                              self.executable.__class__.__name__,
                                              self.executable.fp)

    def _create_disassembler(self):
        '''
        Creates a capstone disassembler instance for this architecture
        :return: None
        '''
        raise NotImplementedError()

    def _gen_ins_map(self):
        '''
        Generates the instruction lookup dictionary
        :return: None
        '''
        raise NotImplementedError()

    def _identify_functions(self):
        '''
        Iterates through instructions and identifies functions by prologues and epilogues
        :return: None
        '''
        raise NotImplementedError()

    def _populate_func_instructions(self):
        '''
        Iterates through all found functions and add instructions inside that function to the Function object
        :return: None
        '''
        insn_addrs = sorted(self.ins_map.keys())

        for f in self.executable.iter_functions():
            try:
                i = insn_addrs.index(f.address)

                while i < len(insn_addrs) and insn_addrs[i] < f.address + f.size:
                    f.instructions.append(self.ins_map[insn_addrs[i]])
                    i += 1
            except ValueError:
                logging.debug('Function {} starts at code outside any executable section.'.format(f))

    def _identify_strings(self):
        '''
        Extracts all strings from the executable and stores them in the strings dict (addr -> string)
        :return: None
        '''
        # https://stackoverflow.com/questions/6804582/extract-strings-from-a-binary-file-in-python
        chars = string.printable
        shortest_run = 2
        regexp = '[%s]{%d,}' % (chars, shortest_run)
        pattern = re.compile(regexp)

        for section in self.executable.iter_string_sections():
            for found_string in pattern.finditer(section.raw):
                vaddr = section.vaddr + found_string.start()
                self.executable.strings[vaddr] = String(found_string.group(), vaddr, self.executable)

    def _identify_bbs(self):
        '''
        Extracts all basic blocks from the executable and stores them in the associated function's bbs list
        :return: None
        '''
        for func in self.executable.iter_functions():
            if func.instructions:
                bb_ends = set([func.instructions[-1].address + func.instructions[-1].size])

                for i in range(len(func.instructions) - 1):
                    cur = func.instructions[i]
                    next = func.instructions[i+1]

                    if cur.is_jump() and cur.operands[0].type == Operand.IMM:
                        bb_ends.add(cur.operands[0].imm)
                        bb_ends.add(next.address)

                bb_ends = sorted(list(bb_ends))
                bb_instructions = []

                for ins in func.instructions:
                    if ins.address == bb_ends[0] and bb_instructions:
                        bb = BasicBlock(func,
                                        bb_instructions[0].address,
                                        bb_instructions[-1].address + bb_instructions[-1].size - bb_instructions[0].address)
                        bb.instructions = bb_instructions
                        func.bbs.append(bb)

                        bb_ends = bb_ends[1:]
                        bb_instructions = [ins]
                    else:
                        bb_instructions.append(ins)

                # There will always be one BB left over which "ends" at the first address of the next function, so be
                # sure to add it

                bb = BasicBlock(func,
                                bb_instructions[0].address,
                                bb_instructions[-1].address + bb_instructions[-1].size - bb_instructions[0].address)
                bb.instructions = bb_instructions
                func.bbs.append(bb)

    def _mark_xrefs(self):
        '''
        Identify all the xrefs from the executable and store them in the xrefs dict (int -> set)
        :return: None
        '''
        for addr, ins in self.ins_map.iteritems():
            for operand in ins.operands:
                if operand.type == Operand.IMM and self.executable.vaddr_binary_offset(operand.imm) is not None:
                    if operand.imm in self.executable.xrefs:
                        self.executable.xrefs[operand.imm].add(addr)
                    else:
                        self.executable.xrefs[operand.imm] = set([addr])

    def analyze(self):
        '''
        Run the analysis subroutines.
        Generates the instruction map, extracts symbol tables, identifies functions/BBs, and "prettifies" instruction op_str's
        :return: None
        '''
        logging.info('Generating instruction map')
        self._gen_ins_map()

        logging.info('Extracting symbol table')
        self.executable._extract_symbol_table()

        logging.info('Identifying functions')
        self._identify_functions()

        # TODO: CFA

        logging.info('Populating function instructions')
        self._populate_func_instructions()
        logging.info('Identifying basic blocks')
        self._identify_bbs()
        logging.info('Marking XRefs')
        self._mark_xrefs()

        logging.info('Identifying strings')
        self._identify_strings()

    def cfg(self):
        '''
        Creates a control flow graph for the binary
        :return: List of tuples that describe the edges of the graph.
        '''
        raise NotImplementedError()
<commit_msg>Update outdated base_executable.cfg docstring<commit_after>import logging
import re
import string
from collections import OrderedDict

from ..constructs import *

class BaseAnalyzer(object):
    '''
    The analyzers are responsible for taking raw instructions from the executable and transforming them
    into higher-level constructs. This includes identifying functions, basic blocks, etc.

    The analyzers also provide some helper methods (ins_*) which are quick ways to determine what an instruction does.
    This can include determining if a instruction is sensitive to location, is a call/jump, etc.
    '''
    def __init__(self, executable):
        self.executable = executable

        # Dictionary of vaddr: instruction for quick lookups.
        # We use an OrderedDict so we can just `for addr in ins_map` and not have to worry about sorting
        self.ins_map = OrderedDict()

    def __repr__(self):
        return '<{} for {} {} \'{}\'>'.format(self.__class__.__name__,
                                              self.executable.architecture,
                                              self.executable.__class__.__name__,
                                              self.executable.fp)

    def _create_disassembler(self):
        '''
        Creates a capstone disassembler instance for this architecture
        :return: None
        '''
        raise NotImplementedError()

    def _gen_ins_map(self):
        '''
        Generates the instruction lookup dictionary
        :return: None
        '''
        raise NotImplementedError()

    def _identify_functions(self):
        '''
        Iterates through instructions and identifies functions by prologues and epilogues
        :return: None
        '''
        raise NotImplementedError()

    def _populate_func_instructions(self):
        '''
        Iterates through all found functions and add instructions inside that function to the Function object
        :return: None
        '''
        insn_addrs = sorted(self.ins_map.keys())

        for f in self.executable.iter_functions():
            try:
                i = insn_addrs.index(f.address)

                while i < len(insn_addrs) and insn_addrs[i] < f.address + f.size:
                    f.instructions.append(self.ins_map[insn_addrs[i]])
                    i += 1
            except ValueError:
                logging.debug('Function {} starts at code outside any executable section.'.format(f))

    def _identify_strings(self):
        '''
        Extracts all strings from the executable and stores them in the strings dict (addr -> string)
        :return: None
        '''
        # https://stackoverflow.com/questions/6804582/extract-strings-from-a-binary-file-in-python
        chars = string.printable
        shortest_run = 2
        regexp = '[%s]{%d,}' % (chars, shortest_run)
        pattern = re.compile(regexp)

        for section in self.executable.iter_string_sections():
            for found_string in pattern.finditer(section.raw):
                vaddr = section.vaddr + found_string.start()
                self.executable.strings[vaddr] = String(found_string.group(), vaddr, self.executable)

    def _identify_bbs(self):
        '''
        Extracts all basic blocks from the executable and stores them in the associated function's bbs list
        :return: None
        '''
        for func in self.executable.iter_functions():
            if func.instructions:
                bb_ends = set([func.instructions[-1].address + func.instructions[-1].size])

                for i in range(len(func.instructions) - 1):
                    cur = func.instructions[i]
                    next = func.instructions[i+1]

                    if cur.is_jump() and cur.operands[0].type == Operand.IMM:
                        bb_ends.add(cur.operands[0].imm)
                        bb_ends.add(next.address)

                bb_ends = sorted(list(bb_ends))
                bb_instructions = []

                for ins in func.instructions:
                    if ins.address == bb_ends[0] and bb_instructions:
                        bb = BasicBlock(func,
                                        bb_instructions[0].address,
                                        bb_instructions[-1].address + bb_instructions[-1].size - bb_instructions[0].address)
                        bb.instructions = bb_instructions
                        func.bbs.append(bb)

                        bb_ends = bb_ends[1:]
                        bb_instructions = [ins]
                    else:
                        bb_instructions.append(ins)

                # There will always be one BB left over which "ends" at the first address of the next function, so be
                # sure to add it

                bb = BasicBlock(func,
                                bb_instructions[0].address,
                                bb_instructions[-1].address + bb_instructions[-1].size - bb_instructions[0].address)
                bb.instructions = bb_instructions
                func.bbs.append(bb)

    def _mark_xrefs(self):
        '''
        Identify all the xrefs from the executable and store them in the xrefs dict (int -> set)
        :return: None
        '''
        for addr, ins in self.ins_map.iteritems():
            for operand in ins.operands:
                if operand.type == Operand.IMM and self.executable.vaddr_binary_offset(operand.imm) is not None:
                    if operand.imm in self.executable.xrefs:
                        self.executable.xrefs[operand.imm].add(addr)
                    else:
                        self.executable.xrefs[operand.imm] = set([addr])

    def analyze(self):
        '''
        Run the analysis subroutines.
        Generates the instruction map, extracts symbol tables, identifies functions/BBs, and "prettifies" instruction op_str's
        :return: None
        '''
        logging.info('Generating instruction map')
        self._gen_ins_map()

        logging.info('Extracting symbol table')
        self.executable._extract_symbol_table()

        logging.info('Identifying functions')
        self._identify_functions()

        # TODO: CFA

        logging.info('Populating function instructions')
        self._populate_func_instructions()
        logging.info('Identifying basic blocks')
        self._identify_bbs()
        logging.info('Marking XRefs')
        self._mark_xrefs()

        logging.info('Identifying strings')
        self._identify_strings()

    def cfg(self):
        '''
        Creates a control flow graph for the binary
        :return: List of CFGEdges that describe the edges of the graph.
        '''
        raise NotImplementedError()
<|endoftext|>


<commit_before>"""
remarkable.

Usage:
  remarkable [options] command <param> <another_params>
  remarkable [options] another-command <param>

  remarkable -h | --help

Options:
  --kw-arg=<kw>         Keyword option description.
  -b --boolean          Boolean option description.
  --debug               Debug.

  -h --help             Show this screen.
"""

from docopt import docopt
import logging

import remarkable

log = logging.getLogger(__name__)

def main():
    arguments = docopt(__doc__, version=changes.__version__)
    debug = arguments['--debug']
    logging.basicConfig(level=logging.DEBUG if debug else logging.INFO)
    log.debug('arguments: %s', arguments)
<commit_msg>Fix cookiecutter bug<commit_after>"""
remarkable.

Usage:
  remarkable [options] command <param> <another_params>
  remarkable [options] another-command <param>

  remarkable -h | --help

Options:
  --kw-arg=<kw>         Keyword option description.
  -b --boolean          Boolean option description.
  --debug               Debug.

  -h --help             Show this screen.
"""

from docopt import docopt
import logging

import remarkable

log = logging.getLogger(__name__)

def main():
    arguments = docopt(__doc__, version=remarkable.__version__)
    debug = arguments['--debug']
    logging.basicConfig(level=logging.DEBUG if debug else logging.INFO)
    log.debug('arguments: %s', arguments)
<|endoftext|>


<commit_before>from rest_framework.views import APIView
from rest_framework import status
from rest_framework import parsers
from rest_framework import renderers
from rest_framework.response import Response
from rest_framework.authtoken.models import Token
from rest_framework.authtoken.serializers import AuthTokenSerializer

class ObtainAuthToken(APIView):
    throttle_classes = ()
    permission_classes = ()
    parser_classes = (parsers.FormParser, parsers.MultiPartParser, parsers.JSONParser,)
    renderer_classes = (renderers.JSONRenderer,)
    model = Token

    def post(self, request):
        serializer = AuthTokenSerializer(data=request.DATA)
        if serializer.is_valid():
            token, created = Token.objects.get_or_create(user=serializer.object['user'])
            return Response({'token': token.key})
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

obtain_auth_token = ObtainAuthToken.as_view()
<commit_msg>ObtainAuthToken pluggable Serializer.<commit_after>from rest_framework.views import APIView
from rest_framework import status
from rest_framework import parsers
from rest_framework import renderers
from rest_framework.response import Response
from rest_framework.authtoken.models import Token
from rest_framework.authtoken.serializers import AuthTokenSerializer

class ObtainAuthToken(APIView):
    throttle_classes = ()
    permission_classes = ()
    parser_classes = (parsers.FormParser, parsers.MultiPartParser, parsers.JSONParser,)
    renderer_classes = (renderers.JSONRenderer,)
    serializer_class = AuthTokenSerializer
    model = Token

    def post(self, request):
        serializer = self.serializer_class(data=request.DATA)
        if serializer.is_valid():
            token, created = Token.objects.get_or_create(user=serializer.object['user'])
            return Response({'token': token.key})
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

obtain_auth_token = ObtainAuthToken.as_view()
<|endoftext|>