<commit_before>#!/usr/bin/env python
#-*- coding:utf-8 -*-
'''
 build.py

 changelog
 2013-12-07[12:01:09]:created
 2013-12-10[22:40:30]:add build time show
 2013-12-15[01:04:04]:less supported
 2013-12-24[21:04:45]:'ignore_parents' setting

 @info yinyong,osx-x64,UTF-8,192.168.1.104,py,/Users/yinyong/work/ursa2/src
 @author yanni4night@gmail.com
 @version 0.0.2
 @since 0.0.1
'''

from conf import C,log,BASE_DIR
import logging
import shutil
import os
import re
import sys
import subprocess
import utils
import time
from render import TokenRender,removeCssDepsDeclaration
from exception import ConfigurationError,DirectoryError
from replace import replace
from timestamp import html_link,html_script,html_img,all_url

RJS_PATH = os.path.join(BASE_DIR,'assets','r.js')
RPL_PATH = os.path.join(BASE_DIR,'assets','rpl.js')

#custom defined yuicompressor
__ycpath = C('yuicompressor')
YC_PATH = None
if __ycpath and utils.isStr( __ycpath ):
    if not os.path.exists( __ycpath ):
        log.warn('%s does not exist'%__ycpath )
    elif not os.path.isfile( __ycpath ) or not __ycpath.endswith('.jar'):
        log.warn('%s is not valid yuicompressor file')
    else:
        YC_PATH = __ycpath

if not YC_PATH:
    YC_PATH = os.path.join(BASE_DIR,'assets','yuicompressor-2.4.8.jar')

class UrsaBuilder(object):
    '''
    build project:
    copy and handle static resources
    generate html with json data
    '''
    @classmethod
    def __init__(self,compress,html,target = None):
        '''
        预先计算一些基础路径
        '''
        log.setLevel(logging.DEBUG)
        self._compress = compress
        self._generate_html = html
        self._target = target
        self._tpl_dir = C('template_dir')
        self._build_dir = C('build_dir')
        self._static_dir = C('static_dir')
        self._js_dir = C('js_dir')
        self._css_dir = C('css_dir')
        self._compile_dir = C('compile_folder') or ''#copy and repalce

        #check dunplicated
        if os.path.relpath(self._build_dir,self._tpl_dir) == '.':
            raise ConfigurationError('template_dir and build_dir are dumplicated')
        elif os.path.relpath(self._build_dir,self._static_dir) == '.':
            raise ConfigurationError('template_dir and static_dir are dumplicated')
        elif self._compile_dir and os.path.relpath(self._compile_dir,self._static_dir) == '.':
            raise ConfigurationError('compile_dir and static_dir are dunplicated')
        elif self._compile_dir and os.path.relpath(self._compile_dir,self._build_dir) == '.':
            raise ConfigurationError('compile_dir and build_dir are dunplicated')
        elif self._compile_dir and os.path.relpath(self._compile_dir,self._tpl_dir) == '.':
            raise ConfigurationError('compile_dir and tpl_dir are dunplicated')

        self._build_static_dir = os.path.join(self._build_dir,self._static_dir)
        self._build_css_dir = os.path.join(self._build_static_dir,self._css_dir,C('css_folder') or '')
        self._build_js_dir = os.path.join(self._build_static_dir,self._js_dir,C('js_folder') or '')
        self._build_tpl_dir = os.path.join(self._build_dir,self._tpl_dir)

        if self._compile_dir:
            self._build_compile_dir = os.path.join(self._build_dir,self._compile_dir)

        self._build_html_dir = os.path.join(self._build_dir,C('html_dir'))

    @classmethod
    def build(self):
        '''
        do build
        '''
        tmbegin = time.time()
        self._check();
        log.info ('copying directories')
        self._dir();
        log.info('handling less...');
        self._less();
        log.info ('handling css...')
        self._css();
        log.info ('handling  javascript...')
        self._js();
        log.info ('handling template...')
        self._tpl();
        if self._generate_html:
            log.info ('handling html...')
            self._html();
        log.info ('Time cost %s s.' % (time.time()-tmbegin) )

    @classmethod
    def _check(self):
        '''
        检查是否具备一个ursa2工程必备的文件和目录
        '''
        require_dirs = [self._tpl_dir,self._static_dir];
        for d in require_dirs:
            if not os.path.exists(d):
                raise DirectoryError('Ursas project requires %s directory'%d)

    @classmethod
    def _dir(self):
        '''
        handle build dir
        '''
        shutil.rmtree(self._build_dir,True)
        os.mkdir(self._build_dir);
        shutil.copytree(self._static_dir,self._build_static_dir)
        shutil.copytree(self._tpl_dir,self._build_tpl_dir)

        if self._compile_dir:
            shutil.copytree(self._compile_dir,self._build_compile_dir)

    @classmethod
    def _less(self):
        '''
        handle less files to css

        '''
        all_less_files = utils.FileSearcher(r'\.less$',self._build_css_dir,relative = False).search()
        for less in all_less_files:
            subprocess.call('lessc %s %s'%(less,re.sub(r"\.less",".css",less)),shell = True)
            os.remove(less)

    @classmethod
    def _css(self):
        '''
        handle css

        r.js会对不同目录下CSS合并时的URL进行修正，因而对于@something@开头的路径会被认为是相对路径，
        产生修正错误，解决方案是先对所有CSS文件进行变量替换，时间戳添加，再由r.js合并。这会降低处理速
        度但可以解决该问题。

        考虑到速度，此过程仅支持在build时进行，开发服务器访问时不能使用。

        所有静态资源路径都应该使用绝对路径，避免在CSS中引用相对路径的图像资源。
        '''

        #搜索所有CSS文件
        all_css_files = utils.FileSearcher(r'\.css$',self._build_css_dir,relative = False).search()

        #替换和加时间戳
        for dst in all_css_files:
            content = utils.readfile(dst)
            content = all_url(content,os.path.dirname(dst))
            content = replace(content,self._target)
            utils.writefile(dst,content)

        #仅对指定的CSS进行r.js合并
        css_modules = C('require_css_modules')
        if not utils.isList(css_modules):
            css_modules = ['main']

        for css in css_modules:
            if not utils.isStr(css):
                continue;
            css = re.sub(r'\/*','',css)
            if not css.endswith('.css'):
                css += '.css'
            css_realpath = os.path.join(self._build_css_dir,css)
            self.build_css(css_realpath,css_realpath)
            continue

    @classmethod
    def build_css(self,src,dst):
        '''
        handle one css src to dst

        合并和按需压缩
        '''
        subprocess.call('node %s -o cssIn=%s out=%s'%(RJS_PATH,src,dst),shell = True)
        if self._compress:
            subprocess.call( 'java -jar ' + YC_PATH + ' --type css --charset ' + C('encoding') + ' ' + dst + ' -o ' + dst , shell = True )

    @classmethod
    def _js(self):
        '''
        handle js

        JS文件不同于CSS，其本身不能引用其它相对路径的静态资源，因此可以实现
        先合并再替换、加时间戳，无需预先处理所有js文件。
        '''
        js_modules = C('require_js_modules') or C('require_modules')
        if not utils.isList(js_modules):
            js_modules = ['main']

        for js in js_modules:
            if not utils.isStr(js):
                continue;
            js = re.sub(r'^\/+','',js)
            if not js.endswith('.js'):
                js += '.js'
            js_realpath = os.path.join(self._build_js_dir,js)
            self.build_js(js_realpath,js_realpath,self._build_js_dir)

    @classmethod
    def build_js(self,src,dst,base_dir):
        '''
        handle one js src to dst

        合并、替换、加时间戳并按需压缩。
        '''
        js = os.path.relpath(src,base_dir)
        subprocess.call( 'node ' + RJS_PATH +' -o name=' + js[0:-3] + ' out='+ dst + ' optimize=none baseUrl='\
            + base_dir , shell = True)
        #repalce
        content = utils.readfile(dst)
        content = replace(content,self._target)
        utils.writefile(dst,content)
        if C('js_ascii_only'):
            subprocess.call( 'node ' + RPL_PATH +' '+dst+' '+dst,shell = True)
        if self._compress:
            subprocess.call( 'java -jar ' + YC_PATH + ' --type js --charset ' + C('encoding') + ' ' + dst + ' -o ' + dst , shell = True )

    @classmethod
    def _tpl(self):
        '''
        handle tempaltes

        模板仅需加时间戳和变量替换。

        这里需要加入额外的{compile_dir}文件夹下的文本文件。
        '''
        fs = utils.FileSearcher(r'\.%s$'%C('template_ext'),self._build_tpl_dir,relative = False)
        tpls = fs.search()
        if self._compile_dir:
            nfs = utils.FileSearcher(r'.+',self._build_compile_dir,relative = False)
            compile_files = nfs.search()
            for f in compile_files:
                if not utils.isBinary(f):
                    tpls.insert(0,f)

        for tpl in tpls:
            content = utils.readfile(tpl)
            #模板的静态资源相对目录应该写死为cwd，即资源路径应该始终是绝对路径
            content = html_link(content,'.')
            content = html_script(content,'.')
            content = html_img(content,'.')
            content = all_url(content,'.')
            content = replace(content,self._target)
            content = removeCssDepsDeclaration(content)
            utils.writefile(tpl,content)

    @classmethod
    def _html(self):
        '''
        generate html

        HTML直接以输出好的模板文件做渲染。

        由于数据原因个别子模板单独渲染会失败，这里用{html_force_output}变量
        可以跳过这类模板。

        TODO：考虑支持require_html_modules
        '''
        fs = utils.FileSearcher(r'\.%s$'%C('template_ext'),self._build_tpl_dir)
        tpls = fs.search()
        for tpl in tpls:
            if C('ignore_parents') and tpl.endswith('parent.'+C('template_ext')):
                continue
            try:
                tr = TokenRender(re.sub(r'\.%s$'%C('template_ext'),'',tpl))
                html = tr.render( build = True)
                target_dir= os.path.join(self._build_html_dir,os.path.dirname(tpl))
                if not os.path.exists(target_dir):
                    os.makedirs(target_dir)
                dst_file = re.sub(r'\.%s$'%C('template_ext'),'.html',os.path.join(self._build_html_dir,tpl))
                utils.writefile(dst_file,html)
            except Exception,e:
                if not C('html_force_output'):
                    raise e
                else:
                    log.error(e)

if __name__ == '__main__':
    builder = UrsaBuilder(True,True,'online')
    builder.build()<commit_msg>Add repalcement of all text files at last<commit_after>#!/usr/bin/env python
#-*- coding:utf-8 -*-
'''
 build.py

 changelog
 2013-12-07[12:01:09]:created
 2013-12-10[22:40:30]:add build time show
 2013-12-15[01:04:04]:less supported
 2013-12-24[21:04:45]:'ignore_parents' setting

 @info yinyong,osx-x64,UTF-8,192.168.1.104,py,/Users/yinyong/work/ursa2/src
 @author yanni4night@gmail.com
 @version 0.0.2
 @since 0.0.1
'''

from conf import C,log,BASE_DIR
import logging
import shutil
import os
import re
import sys
import subprocess
import utils
import time
from render import TokenRender,removeCssDepsDeclaration
from exception import ConfigurationError,DirectoryError
from replace import replace
from timestamp import html_link,html_script,html_img,all_url

RJS_PATH = os.path.join(BASE_DIR,'assets','r.js')
RPL_PATH = os.path.join(BASE_DIR,'assets','rpl.js')

#custom defined yuicompressor
__ycpath = C('yuicompressor')
YC_PATH = None
if __ycpath and utils.isStr( __ycpath ):
    if not os.path.exists( __ycpath ):
        log.warn('%s does not exist'%__ycpath )
    elif not os.path.isfile( __ycpath ) or not __ycpath.endswith('.jar'):
        log.warn('%s is not valid yuicompressor file')
    else:
        YC_PATH = __ycpath

if not YC_PATH:
    YC_PATH = os.path.join(BASE_DIR,'assets','yuicompressor-2.4.8.jar')

class UrsaBuilder(object):
    '''
    build project:
    copy and handle static resources
    generate html with json data
    '''
    @classmethod
    def __init__(self,compress,html,target = None):
        '''
        预先计算一些基础路径
        '''
        log.setLevel(logging.DEBUG)
        self._compress = compress
        self._generate_html = html
        self._target = target
        self._tpl_dir = C('template_dir')
        self._build_dir = C('build_dir')
        self._static_dir = C('static_dir')
        self._js_dir = C('js_dir')
        self._css_dir = C('css_dir')
        self._compile_dir = C('compile_folder') or ''#copy and repalce

        #check dunplicated
        if os.path.relpath(self._build_dir,self._tpl_dir) == '.':
            raise ConfigurationError('template_dir and build_dir are dumplicated')
        elif os.path.relpath(self._build_dir,self._static_dir) == '.':
            raise ConfigurationError('template_dir and static_dir are dumplicated')
        elif self._compile_dir and os.path.relpath(self._compile_dir,self._static_dir) == '.':
            raise ConfigurationError('compile_dir and static_dir are dunplicated')
        elif self._compile_dir and os.path.relpath(self._compile_dir,self._build_dir) == '.':
            raise ConfigurationError('compile_dir and build_dir are dunplicated')
        elif self._compile_dir and os.path.relpath(self._compile_dir,self._tpl_dir) == '.':
            raise ConfigurationError('compile_dir and tpl_dir are dunplicated')

        self._build_static_dir = os.path.join(self._build_dir,self._static_dir)
        self._build_css_dir = os.path.join(self._build_static_dir,self._css_dir,C('css_folder') or '')
        self._build_js_dir = os.path.join(self._build_static_dir,self._js_dir,C('js_folder') or '')
        self._build_tpl_dir = os.path.join(self._build_dir,self._tpl_dir)

        if self._compile_dir:
            self._build_compile_dir = os.path.join(self._build_dir,self._compile_dir)

        self._build_html_dir = os.path.join(self._build_dir,C('html_dir'))

    @classmethod
    def build(self):
        '''
        do build
        '''
        tmbegin = time.time()
        self._check();
        log.info ('copying directories')
        self._dir();
        log.info('handling less...');
        self._less();
        log.info ('handling css...')
        self._css();
        log.info ('handling  javascript...')
        self._js();
        log.info ('handling template...')
        self._tpl();
        if self._generate_html:
            log.info ('handling html...')
            self._html();
        log.info ('replacing all token')
        self._replace();
        log.info ('Time cost %s s.' % (time.time()-tmbegin) )

    @classmethod
    def _check(self):
        '''
        检查是否具备一个ursa2工程必备的文件和目录
        '''
        require_dirs = [self._tpl_dir,self._static_dir];
        for d in require_dirs:
            if not os.path.exists(d):
                raise DirectoryError('Ursas project requires %s directory'%d)

    @classmethod
    def _dir(self):
        '''
        handle build dir
        '''
        shutil.rmtree(self._build_dir,True)
        os.mkdir(self._build_dir);
        shutil.copytree(self._static_dir,self._build_static_dir)
        shutil.copytree(self._tpl_dir,self._build_tpl_dir)

        if self._compile_dir:
            shutil.copytree(self._compile_dir,self._build_compile_dir)

    @classmethod
    def _less(self):
        '''
        handle less files to css

        '''
        all_less_files = utils.FileSearcher(r'\.less$',self._build_css_dir,relative = False).search()
        for less in all_less_files:
            subprocess.call('lessc %s %s'%(less,re.sub(r"\.less",".css",less)),shell = True)
            os.remove(less)

    @classmethod
    def _css(self):
        '''
        handle css

        r.js会对不同目录下CSS合并时的URL进行修正，因而对于@something@开头的路径会被认为是相对路径，
        产生修正错误，解决方案是先对所有CSS文件进行变量替换，时间戳添加，再由r.js合并。这会降低处理速
        度但可以解决该问题。

        考虑到速度，此过程仅支持在build时进行，开发服务器访问时不能使用。

        所有静态资源路径都应该使用绝对路径，避免在CSS中引用相对路径的图像资源。
        '''

        #搜索所有CSS文件
        all_css_files = utils.FileSearcher(r'\.css$',self._build_css_dir,relative = False).search()

        #替换和加时间戳
        for dst in all_css_files:
            content = utils.readfile(dst)
            content = all_url(content,os.path.dirname(dst))
            content = replace(content,self._target)
            utils.writefile(dst,content)

        #仅对指定的CSS进行r.js合并
        css_modules = C('require_css_modules')
        if not utils.isList(css_modules):
            css_modules = ['main']

        for css in css_modules:
            if not utils.isStr(css):
                continue;
            css = re.sub(r'\/*','',css)
            if not css.endswith('.css'):
                css += '.css'
            css_realpath = os.path.join(self._build_css_dir,css)
            self.build_css(css_realpath,css_realpath)
            continue

    @classmethod
    def build_css(self,src,dst):
        '''
        handle one css src to dst

        合并和按需压缩
        '''
        subprocess.call('node %s -o cssIn=%s out=%s'%(RJS_PATH,src,dst),shell = True)
        if self._compress:
            subprocess.call( 'java -jar ' + YC_PATH + ' --type css --charset ' + C('encoding') + ' ' + dst + ' -o ' + dst , shell = True )

    @classmethod
    def _js(self):
        '''
        handle js

        JS文件不同于CSS，其本身不能引用其它相对路径的静态资源，因此可以实现
        先合并再替换、加时间戳，无需预先处理所有js文件。
        '''
        js_modules = C('require_js_modules') or C('require_modules')
        if not utils.isList(js_modules):
            js_modules = ['main']

        for js in js_modules:
            if not utils.isStr(js):
                continue;
            js = re.sub(r'^\/+','',js)
            if not js.endswith('.js'):
                js += '.js'
            js_realpath = os.path.join(self._build_js_dir,js)
            self.build_js(js_realpath,js_realpath,self._build_js_dir)

    @classmethod
    def build_js(self,src,dst,base_dir):
        '''
        handle one js src to dst

        合并、替换、加时间戳并按需压缩。
        '''
        js = os.path.relpath(src,base_dir)
        subprocess.call( 'node ' + RJS_PATH +' -o name=' + js[0:-3] + ' out='+ dst + ' optimize=none baseUrl='\
            + base_dir , shell = True)
        #repalce
        content = utils.readfile(dst)
        content = replace(content,self._target)
        utils.writefile(dst,content)
        if C('js_ascii_only'):
            subprocess.call( 'node ' + RPL_PATH +' '+dst+' '+dst,shell = True)
        if self._compress:
            subprocess.call( 'java -jar ' + YC_PATH + ' --type js --charset ' + C('encoding') + ' ' + dst + ' -o ' + dst , shell = True )

    @classmethod
    def _tpl(self):
        '''
        handle tempaltes

        模板仅需加时间戳和变量替换。

        这里需要加入额外的{compile_dir}文件夹下的文本文件。
        '''
        fs = utils.FileSearcher(r'\.%s$'%C('template_ext'),self._build_tpl_dir,relative = False)
        tpls = fs.search()
        if self._compile_dir:
            nfs = utils.FileSearcher(r'.+',self._build_compile_dir,relative = False)
            compile_files = nfs.search()
            for f in compile_files:
                if not utils.isBinary(f):
                    tpls.insert(0,f)

        for tpl in tpls:
            content = utils.readfile(tpl)
            #模板的静态资源相对目录应该写死为cwd，即资源路径应该始终是绝对路径
            content = html_link(content,'.')
            content = html_script(content,'.')
            content = html_img(content,'.')
            content = all_url(content,'.')
            content = replace(content,self._target)
            content = removeCssDepsDeclaration(content)
            utils.writefile(tpl,content)

    @classmethod
    def _replace(self):
        '''
        替换所有文本的变量
        '''
        files = utils.FileSearcher(r'.+',self._build_dir).search()
        for f in files:
            f = os.path.join(self._build_dir,f)
            if not utils.isBinary(f):
                content = utils.readfile(f)
                content = replace(content,self._target)
                utils.writefile(f,content)

    @classmethod
    def _html(self):
        '''
        generate html

        HTML直接以输出好的模板文件做渲染。

        由于数据原因个别子模板单独渲染会失败，这里用{html_force_output}变量
        可以跳过这类模板。

        TODO：考虑支持require_html_modules
        '''
        fs = utils.FileSearcher(r'\.%s$'%C('template_ext'),self._build_tpl_dir)
        tpls = fs.search()
        for tpl in tpls:
            if C('ignore_parents') and tpl.endswith('parent.'+C('template_ext')):
                continue
            try:
                tr = TokenRender(re.sub(r'\.%s$'%C('template_ext'),'',tpl))
                html = tr.render( build = True)
                target_dir= os.path.join(self._build_html_dir,os.path.dirname(tpl))
                if not os.path.exists(target_dir):
                    os.makedirs(target_dir)
                dst_file = re.sub(r'\.%s$'%C('template_ext'),'.html',os.path.join(self._build_html_dir,tpl))
                utils.writefile(dst_file,html)
            except Exception,e:
                if not C('html_force_output'):
                    raise e
                else:
                    log.error(e)

if __name__ == '__main__':
    builder = UrsaBuilder(True,True,'online')
    builder._replace()<|endoftext|>


<commit_before><commit_msg>Fix for [enter] not moving to left-most cell<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import json
import codecs
from adsputils import u2asc
from default import BaseXmlToDictParser

class NoSchemaException(Exception):
    pass

class WrongSchemaException(Exception):
    pass

class UnparseableException(Exception):
    pass

class JATSParser(BaseXmlToDictParser):

    def __init__(self):
    # make sure we are utf-8 clean on stdout, stderr
        self.JATS_SCHEMA = ""

    def resource_dict(self, fp, **kwargs):
        d = self.xmltodict(fp, **kwargs)
# this returns the root of a JATS article, which is the element

        r = d.get('article',{})
        return r

    def _attribs(self, r, **kwargs):
        kl = self._dict(r).keys()
        attribs = {}
        for k in kl:
            if k[0] == '@':
                attribs[k] = r[k]
        return attribs

    def parse(self, fp, **kwargs):

        output_metadata=dict()

        r = self.resource_dict(fp, **kwargs)

        article_attr = self._attribs(r)
        front_meta = r.get('front')
        back_meta = r.get('back')

        try:
            front_keys, back_keys = (front_meta.keys(),back_meta.keys())
        except AttributeError:
            return
        else:
            article_meta = front_meta.get('article-meta')
            journal_meta = front_meta.get('journal-meta')
            try:
                article_keys, journal_keys = (article_meta.keys(),back_meta.keys())
            except AttributeError:
                return
            else:

# Title
                try:
                    output_metadata['title'] = article_meta['title-group']['article-title']
                except KeyError:
                    pass

# Abstract
                try:
                    output_metadata['abstract'] = article_meta['abstract']['p']
                except KeyError:
                    pass

# Author(s)
                try:
                    output_metadata['authors'] = article_meta['contrib-group']['contrib']
                    for a in output_metadata['authors']:
                        if self._attr(a,'contrib-type') == 'author':
                            print "Author: %s, %s"%(a['name']['surname'],a['name']['given-names'])
                except KeyError:
                    pass

        return output_metadata

if __name__ == "__main__":

    sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
    sys.stderr = codecs.getwriter('utf-8')(sys.stderr)

    jatsx = JATSParser()

    woo = None
    with open('/Users/mtempleton/adsaps.work/fulltext.xml','rU') as fp:
        woo = jatsx.parse(fp)

    print(woo)
<commit_msg> 	modified:   pyingest/parsers/jats.py<commit_after>#!/usr/bin/env python

import sys
import json
import codecs
from adsputils import u2asc
from default import BaseXmlToDictParser

class NoSchemaException(Exception):
    pass

class WrongSchemaException(Exception):
    pass

class UnparseableException(Exception):
    pass

class JATSParser(BaseXmlToDictParser):

    def __init__(self):
    # make sure we are utf-8 clean on stdout, stderr
        self.JATS_SCHEMA = ""

    def resource_dict(self, fp, **kwargs):
        d = self.xmltodict(fp, **kwargs)
# this returns the root of a JATS article, which is the element

        r = d.get('article',{})
        return r

    def _attribs(self, r, **kwargs):
        kl = self._dict(r).keys()
        attribs = {}
        for k in kl:
            if k[0] == '@':
                attribs[k] = r[k]
        return attribs

    def parse(self, fp, **kwargs):

        output_metadata=dict()

        r = self.resource_dict(fp, **kwargs)

        article_attr = self._attribs(r)
        front_meta = r.get('front')
        back_meta = r.get('back')

        try:
            front_keys, back_keys = (front_meta.keys(),back_meta.keys())
        except AttributeError:
            return
        else:
            article_meta = front_meta.get('article-meta')
            journal_meta = front_meta.get('journal-meta')
            try:
                article_keys, journal_keys = (article_meta.keys(),back_meta.keys())
            except AttributeError:
                return
            else:

# Title
                try:
                    output_metadata['title'] = article_meta['title-group']['article-title']
                except KeyError:
                    pass

# Abstract
                try:
                    output_metadata['abstract'] = article_meta['abstract']['p']
                except KeyError:
                    pass

# Author(s)
                authors = []
                try:
                    author_meta = article_meta['contrib-group']['contrib']
                    for a in author_meta:
                        if self._attr(a,'contrib-type') == 'author':
                            authors.append(a['name']['surname']+", "+a['name']['given-names'])
                    if len(authors) > 0:
                        output_metadata['authors'] = '; '.join(authors)
                except KeyError:
                    pass

        return output_metadata

if __name__ == "__main__":

    sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
    sys.stderr = codecs.getwriter('utf-8')(sys.stderr)

    jatsx = JATSParser()

    woo = None
    with open('/Users/mtempleton/adsaps.work/fulltext.xml','rU') as fp:
        woo = jatsx.parse(fp)

    print(woo)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import json
import codecs
from adsputils import u2asc
from default import BaseXmlToDictParser
from xmltodict import unparse

class NoSchemaException(Exception):
    pass

class WrongSchemaException(Exception):
    pass

class UnparseableException(Exception):
    pass

class JATSParser(BaseXmlToDictParser):

    def __init__(self):
        pass

    def resource_dict(self, fp, **kwargs):
        d = self.xmltodict(fp, **kwargs)
# this returns the root of a JATS article, which is the element

        r = d.get('article',{})
        return r

    def _attribs(self, r, **kwargs):
        kl = self._dict(r).keys()
        attribs = {}
        for k in kl:
            if k[0] == '@':
                attribs[k] = r[k]
        return attribs

    def _gettext(self, r):
        if isinstance(r,list):
            return "; ".join(self._gettext(x) for x in r)
        elif isinstance(r,tuple):
            return ": ".join(self._gettext(x) for x in r)
        elif isinstance(r,dict):
            return self._gettext(r.items())
        elif isinstance(r,basestring):
            return r

    def parse(self, fp, **kwargs):

        output_metadata=dict()

# The storage of the raw jats data is a kludge to deal with cases where you
# need to deal with titles and/or abstracts with <inline-formula>s.  See
# the Title and Abstract sections below.

        raw_jats_data = fp.read()

        r = self.resource_dict(raw_jats_data, **kwargs)

#       article_attr = self._attribs(r)
        front_meta = r.get('front')
#       back_meta = r.get('back')

        try:
#           front_keys, back_keys = (front_meta.keys(),back_meta.keys())
            front_keys = front_meta.keys()
        except AttributeError:
            raise WrongSchemaException("Could not parse front and back matter.")
        else:
            article_meta = front_meta.get('article-meta')
            journal_meta = front_meta.get('journal-meta')
            try:
                article_keys, journal_keys = (article_meta.keys(),journal_meta.keys())
            except AttributeError:
                raise WrongSchemaException("Could not parse article and journal metadata.")
            else:
                base_metadata = {}

#Title:
                try:
                    title = article_meta['title-group']['article-title']
                except:
                    pass
                else:
                    if isinstance(title,unicode):
                        base_metadata['title'] = title
                    else:
                        base_metadata['title'] = raw_jats_data.split('<title-group>')[1].split('
')[1].split('
')[0].decode('utf-8')

#Abstract:
                try:
                    abstract = article_meta['abstract']['p']
                except:
                    pass
                else:
                    if isinstance(abstract,unicode):
                        base_metadata['abstract'] = abstract
                    else:
                        base_metadata['abstract'] = raw_jats_data.split('<abstract>')[1].split('
')[1].split('
')[0].decode('utf-8')

#Authors and Affiliations:
#Affiliations first:
                affils = {}
#Affil notes first:
                try:
                    notes_meta = article_meta['author-notes']
                except KeyError:
                    pass
                else:
                    try:
                        note = notes_meta['fn']
                    except KeyError:
                        pass
                    else:
                        if type(note) != list:
                            note = [note]
                        for n in note:
                            try:
                                affils[self._attr(n,'id')] = self._gettext(n['p'])
                            except:
                                try:
                                    affils[self._attr(n,'id')] = self._gettext(n)
                                except:
                                    affils[self._attr(n,'id')] = ''

                try:
                    contrib_group = article_meta['contrib-group']
                except:
                    print "could not get article_meta['contrib-group']"
                else:
                    if type(contrib_group) == type(list()):
                        print "article_meta['contrib-group'] is a list!"

                    else:
                        print "article_meta['contrib-group'] is NOT a list!"

#Author affs next:
                        try:
                            affil_meta = contrib_group['aff']
                        except KeyError:
                            pass
                        else:
                            if type(affil_meta) == type(list()):
                                for a in affil_meta:
                                    try:
                                        a['institution']
                                    except KeyError:
                                        a['institution'] = ''
                                    try:
                                        print('inst_a:',type(a['institution']),'text_a:',type(self._text(a)))
                                        if type(a['institution']) == type(list()):
                                            affils[self._attr(a,'id')] = ', '.join(a['institution']) + ', ' + self._text(a)
                                        else:
                                            affils[self._attr(a,'id')] = a['institution'] + ', ' + self._text(a)
                                    except KeyError:
                                        pass
                            else:
                                try:
                                    affil_meta['institution']
                                except KeyError:
                                    affil_meta['institution'] = ''
                                try:
                                    print('inst_a:',type(affil_meta['institution']),'text_a:',type(self._text(affil_meta)))
                                    if type(affil_meta['institution']) == type(list()):
                                        affils[self._attr(affil_meta,'id')] = ', '.join(affil_meta['institution']) + ', ' + self._text(affil_meta)
                                    else:
                                        affils[self._attr(affil_meta,'id')] = affil_meta['institution'] + ', ' + self._text(affil_meta)
                                except KeyError:
                                    pass

#Authors and note keys
                        try:
                            author_meta = contrib_group['contrib']
                        except KeyError:
                            pass
                        else:
                            base_metadata['authors'] = list()
                            base_metadata['affiliations'] = list()
                            for a in author_meta:
                                if self._attr(a,'contrib-type') == 'author':
                                    try:
                                        pre = a['name']['prefix']+' '
                                    except:
                                        pre = ''
                                    try:
                                        suf = ' '+a['name']['suffix']
                                    except:
                                        suf = ''
                                    try:
                                        sur = a['name']['surname']
                                    except:
                                        sur = 'Anonymous'
                                    try:
                                        giv = a['name']['given-names']
                                    except:
                                        giv = ''
                                    if (pre+giv+suf) == '':
                                        base_metadata['authors'].append(sur)
                                    else:
                                        base_metadata['authors'].append(sur+", "+pre+giv+suf)
                                    try:
                                        affs = self._array(a['xref'])
                                        aid = self._gettext([self._attr(x,'rid') for x in affs]).replace(';',' ').split()
                                    except:
                                        base_metadata['affiliations'].append('')
                                    else:
                                        aff_string = "; ".join([affils[x] for x in aid])
                                        base_metadata['affiliations'].append(aff_string)
                            base_metadata['authors'] = "; ".join(base_metadata['authors'])

#Keywords:
                try:
                    art_cats = article_meta['article-categories']['subj-group']
                except:
                    pass
                else:
                    if type(art_cats) == type(list()):
                        for c in art_cats:
                            if c['@subj-group-type'] == 'toc-minor':
                                base_metadata['keywords'] = c['subject'].replace(',',';')
                    else:
                        if art_cats['@subj-group-type'] == 'toc-minor':
                            base_metadata['keywords'] = art_cats['subject'].replace(',',';')

#Pubdate:
                try:
                    pub_dates = article_meta['pub-date']
                except:
                    pass
                else:
                    if(type(pub_dates)) == type(list()):
                        for d in pub_dates:
                            a = self._attr(d,'publication-format')
                            b = self._attr(d,'pub-type')
                            if (a == 'print' or b == 'ppub'):
                                base_metadata['pubdate'] = self._attr(d,'iso-8601-date')
                                if base_metadata['pubdate'][-2:] == '00':
                                    base_metadata['pubdate'] = base_metadata['pubdate'][0:-2]+'01'
                            else:
                                if (b == 'epub'):
                                    base_metadata['pubdate'] = self._attr(d,'iso-8601-date')
                                    if base_metadata['pubdate'][-2:] == '00':
                                        base_metadata['pubdate'] = base_metadata['pubdate'][0:-2]+'01'
                    else:
                        a = self._attr(pub_dates,'publication-format')
                        b = self._attr(pub_dates,'pub-type')
                        if (a == 'print' or b == 'ppub'):
                            base_metadata['pubdate'] = self._attr(pub_dates,'iso-8601-date')
                            if base_metadata['pubdate'][-2:] == '00':
                                base_metadata['pubdate'] = base_metadata['pubdate'][0:-2]+'01'
                        else:
                            if (b == 'epub'):
                                base_metadata['pubdate'] = self._attr(pub_dates,'iso-8601-date')
                                if base_metadata['pubdate'][-2:] == '00':
                                    base_metadata['pubdate'] = base_metadata['pubdate'][0:-2]+'01'

#DOI:
                try:
                    id_meta = article_meta['article-id']
                    try:
                        if self._attr(id_meta,'pub-id-type') == 'doi':
                            base_metadata['properties'] = {'DOI': 'doi:'+self._text(id_meta)}
                    except:
                        pass
                except:
                    pass

#Journal name:
                try:
                    base_metadata['publication'] = journal_meta['journal-title-group']['journal-title']
                    if type(base_metadata['publication']) == type(list()):
                        base_metadata['publication'] = base_metadata['publication'][0]
                except:
                    pass
#Journal ID:
                try:
                    for i in journal_meta['journal-id']:
                        if self._attr(i,'journal-id-type') == 'publisher-id':
                            base_metadata['pub-id'] = self._text(i)
                except:
                    pass

#Volume:
                try:
                    base_metadata['volume'] = article_meta['volume']
                except:
                    pass

#Issue:
                try:
                    base_metadata['issue'] = article_meta['issue']
                except:
                    pass

#Pages:
                try:
                    fpage = article_meta['fpage']
                except:
                    try:
                        fpage = article_meta['elocation-id']
                    except:
                        pass
                try:
                    fpage
                except NameError:
                    pass
                else:
                    try:
                        lpage = article_meta['lpage']
                    except KeyError:
                        pass
                    else:
                        if lpage == fpage:
                            del lpage
                    try:
                        lpage
                    except NameError:
                        base_metadata['page'] = fpage
                    else:
                        base_metadata['page'] = fpage + "-" + lpage

                output_metadata = base_metadata

        return output_metadata
#
#
#
#
#
#if __name__ == "__main__":
#
#    sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
#    sys.stderr = codecs.getwriter('utf-8')(sys.stderr)
#
#    jatsx = JATSParser()
#
#    woo = None
#    with open('/Users/mtempleton/adsaps.work/fulltext.xml','rU') as fp:
#        woo = jatsx.parse(fp)
#
##   print(woo)
<commit_msg> 	modified:   jats.py<commit_after>#!/usr/bin/env python

import sys
import json
import codecs
from adsputils import u2asc
from default import BaseXmlToDictParser
from xmltodict import unparse

class NoSchemaException(Exception):
    pass

class WrongSchemaException(Exception):
    pass

class UnparseableException(Exception):
    pass

class JATSParser(BaseXmlToDictParser):

    def __init__(self):
        pass

    def resource_dict(self, fp, **kwargs):
        d = self.xmltodict(fp, **kwargs)
# this returns the root of a JATS article, which is the element

        r = d.get('article',{})
        return r

    def _attribs(self, r, **kwargs):
        kl = self._dict(r).keys()
        attribs = {}
        for k in kl:
            if k[0] == '@':
                attribs[k] = r[k]
        return attribs

    def _gettext(self, r):
        if isinstance(r,list):
            return "; ".join(self._gettext(x) for x in r)
        elif isinstance(r,tuple):
            return ": ".join(self._gettext(x) for x in r)
        elif isinstance(r,dict):
            return self._gettext(r.items())
        elif isinstance(r,basestring):
            return r

    def parse(self, fp, **kwargs):

        output_metadata=dict()

# The storage of the raw jats data is a kludge to deal with cases where you
# need to deal with titles and/or abstracts with <inline-formula>s.  See
# the Title and Abstract sections below.

        raw_jats_data = fp.read()

        r = self.resource_dict(raw_jats_data, **kwargs)

#       article_attr = self._attribs(r)
        front_meta = r.get('front')
#       back_meta = r.get('back')

        try:
#           front_keys, back_keys = (front_meta.keys(),back_meta.keys())
            front_keys = front_meta.keys()
        except AttributeError:
            raise WrongSchemaException("Could not parse front and back matter.")
        else:
            article_meta = front_meta.get('article-meta')
            journal_meta = front_meta.get('journal-meta')
            try:
                article_keys, journal_keys = (article_meta.keys(),journal_meta.keys())
            except AttributeError:
                raise WrongSchemaException("Could not parse article and journal metadata.")
            else:
                base_metadata = {}

#Title:
                try:
                    title = article_meta['title-group']['article-title']
                except:
                    pass
                else:
                    if isinstance(title,unicode):
                        base_metadata['title'] = title
                    else:
                        base_metadata['title'] = raw_jats_data.split('<title-group>')[1].split('
')[1].split('
')[0].decode('utf-8')

#Abstract:
                try:
                    abstract = article_meta['abstract']['p']
                except:
                    pass
                else:
                    if isinstance(abstract,unicode):
                        base_metadata['abstract'] = abstract
                    else:
                        base_metadata['abstract'] = raw_jats_data.split('<abstract>')[1].split('
')[1].split('
')[0].decode('utf-8')

#Authors and Affiliations:
#Affiliations first:
                affils = {}
#Affil notes first:
                try:
                    notes_meta = article_meta['author-notes']
                except KeyError:
                    pass
                else:
                    try:
                        note = notes_meta['fn']
                    except KeyError:
                        pass
                    else:
                        if type(note) != list:
                            note = [note]
                        for n in note:
                            try:
                                affils[self._attr(n,'id')] = self._gettext(n['p'])
                            except:
                                try:
                                    affils[self._attr(n,'id')] = self._gettext(n)
                                except:
                                    affils[self._attr(n,'id')] = ''

                try:
                    contrib_group = article_meta['contrib-group']
                except:
                    print "could not get article_meta['contrib-group']"
                else:
                    if type(contrib_group) == type(list()):
                        print "article_meta['contrib-group'] is a list!"

# YOU NEED TO DO SOMETHING HERE IF IT IS A LIST!

                    else:
                        print "article_meta['contrib-group'] is NOT a list!"

#Author affs next:
                        try:
                            affil_meta = contrib_group['aff']
                        except KeyError:
                            pass
                        else:
                            if type(affil_meta) == type(list()):
                                for a in affil_meta:
                                    try:
                                        a['institution']
                                    except KeyError:
                                        a['institution'] = ''
                                    try:
                                        print('inst_a:',type(a['institution']),'text_a:',type(self._text(a)))
                                        if type(a['institution']) == type(list()):
                                            affils[self._attr(a,'id')] = ', '.join(a['institution']) + ', ' + self._text(a)
                                        else:
                                            affils[self._attr(a,'id')] = a['institution'] + ', ' + self._text(a)
                                    except KeyError:
                                        pass
                            else:
                                try:
                                    affil_meta['institution']
                                except KeyError:
                                    affil_meta['institution'] = ''
                                try:
                                    print('inst_a:',type(affil_meta['institution']),'text_a:',type(self._text(affil_meta)))
                                    if type(affil_meta['institution']) == type(list()):
                                        affils[self._attr(affil_meta,'id')] = ', '.join(affil_meta['institution']) + ', ' + self._text(affil_meta)
                                    else:
                                        affils[self._attr(affil_meta,'id')] = affil_meta['institution'] + ', ' + self._text(affil_meta)
                                except KeyError:
                                    pass

#Authors and note keys
                        try:
                            author_meta = contrib_group['contrib']
                        except KeyError:
                            pass
                        else:
                            base_metadata['authors'] = list()
                            base_metadata['affiliations'] = list()
                            for a in author_meta:
                                if self._attr(a,'contrib-type') == 'author':
                                    try:
                                        pre = a['name']['prefix']+' '
                                    except:
                                        pre = ''
                                    try:
                                        suf = ' '+a['name']['suffix']
                                    except:
                                        suf = ''
                                    try:
                                        sur = a['name']['surname']
                                    except:
                                        sur = 'Anonymous'
                                    try:
                                        giv = a['name']['given-names']
                                    except:
                                        giv = ''
                                    if (pre+giv+suf) == '':
                                        base_metadata['authors'].append(sur)
                                    else:
                                        base_metadata['authors'].append(sur+", "+pre+giv+suf)
                                    try:
                                        affs = self._array(a['xref'])
                                        aid = self._gettext([self._attr(x,'rid') for x in affs]).replace(';',' ').split()
                                    except:
                                        base_metadata['affiliations'].append('')
                                    else:
                                        aff_string = "; ".join([affils[x] for x in aid])
                                        base_metadata['affiliations'].append(aff_string)
                            base_metadata['authors'] = "; ".join(base_metadata['authors'])

#Keywords:
                try:
                    art_cats = article_meta['article-categories']['subj-group']
                except:
                    pass
                else:
                    if type(art_cats) == type(list()):
                        for c in art_cats:
                            if c['@subj-group-type'] == 'toc-minor':
                                base_metadata['keywords'] = c['subject'].replace(',',';')
                    else:
                        if art_cats['@subj-group-type'] == 'toc-minor':
                            base_metadata['keywords'] = art_cats['subject'].replace(',',';')

#Pubdate:
                try:
                    pub_dates = article_meta['pub-date']
                except:
                    pass
                else:
                    if(type(pub_dates)) == type(list()):
                        for d in pub_dates:
                            a = self._attr(d,'publication-format')
                            b = self._attr(d,'pub-type')
                            if (a == 'print' or b == 'ppub'):
                                base_metadata['pubdate'] = self._attr(d,'iso-8601-date')
                                if base_metadata['pubdate'][-2:] == '00':
                                    base_metadata['pubdate'] = base_metadata['pubdate'][0:-2]+'01'
                            else:
                                if (b == 'epub'):
                                    base_metadata['pubdate'] = self._attr(d,'iso-8601-date')
                                    if base_metadata['pubdate'][-2:] == '00':
                                        base_metadata['pubdate'] = base_metadata['pubdate'][0:-2]+'01'
                    else:
                        a = self._attr(pub_dates,'publication-format')
                        b = self._attr(pub_dates,'pub-type')
                        if (a == 'print' or b == 'ppub'):
                            base_metadata['pubdate'] = self._attr(pub_dates,'iso-8601-date')
                            if base_metadata['pubdate'][-2:] == '00':
                                base_metadata['pubdate'] = base_metadata['pubdate'][0:-2]+'01'
                        else:
                            if (b == 'epub'):
                                base_metadata['pubdate'] = self._attr(pub_dates,'iso-8601-date')
                                if base_metadata['pubdate'][-2:] == '00':
                                    base_metadata['pubdate'] = base_metadata['pubdate'][0:-2]+'01'

#DOI:
                try:
                    id_meta = article_meta['article-id']
                    try:
                        if self._attr(id_meta,'pub-id-type') == 'doi':
                            base_metadata['properties'] = {'DOI': 'doi:'+self._text(id_meta)}
                    except:
                        pass
                except:
                    pass

#Journal name:
                try:
                    base_metadata['publication'] = journal_meta['journal-title-group']['journal-title']
                    if type(base_metadata['publication']) == type(list()):
                        base_metadata['publication'] = base_metadata['publication'][0]
                except:
                    pass
#Journal ID:
                try:
                    for i in journal_meta['journal-id']:
                        if self._attr(i,'journal-id-type') == 'publisher-id':
                            base_metadata['pub-id'] = self._text(i)
                except:
                    pass

#Volume:
                try:
                    base_metadata['volume'] = article_meta['volume']
                except:
                    pass

#Issue:
                try:
                    base_metadata['issue'] = article_meta['issue']
                except:
                    pass

#Pages:
                try:
                    fpage = article_meta['fpage']
                except:
                    try:
                        fpage = article_meta['elocation-id']
                    except:
                        pass
                try:
                    fpage
                except NameError:
                    pass
                else:
                    try:
                        lpage = article_meta['lpage']
                    except KeyError:
                        pass
                    else:
                        if lpage == fpage:
                            del lpage
                    try:
                        lpage
                    except NameError:
                        base_metadata['page'] = fpage
                    else:
                        base_metadata['page'] = fpage + "-" + lpage

                output_metadata = base_metadata

        return output_metadata
#
#
#
#
#
#if __name__ == "__main__":
#
#    sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
#    sys.stderr = codecs.getwriter('utf-8')(sys.stderr)
#
#    jatsx = JATSParser()
#
#    woo = None
#    with open('/Users/mtempleton/adsaps.work/fulltext.xml','rU') as fp:
#        woo = jatsx.parse(fp)
#
##   print(woo)
<|endoftext|>


<commit_before>####python MFK_final.py <model> <nMoments> <outputfile>

from time import time
import sys
import os

from sympy import Matrix, diff, Symbol, Subs, Eq, var, simplify
from fcount import fcount
from sympy import Matrix, diff, Symbol, Subs, Eq, var, simplify
from sympy import S as F
from sympy.solvers import solve
from TaylorExpansion import TaylorExpansion
from centralmoments import eq_centralmoments
from raw_to_central import raw_to_central
from sympy import latex

#############################################################################
#   MFK_final takes number of moments as input, and produces central
#   moment equations using the specified model
#############################################################################

def get_and_check_model():
    from model import model

    (S,a,nreactions,nvariables,ymat,Mumat, c) = model()

    #Delete temporary model file
    os.system('rm model.py*')
    #TODO use assertions instead of `if else`
    #Check stoichiometry matrix
    if S.cols==nreactions and S.rows==nvariables:
        print "S=okay"
    elif S.cols!=nreactions:
        print "Wrong number of reactions in S"
    elif S.rows!=nvariables:
        print "Wrong number of variables in S"

    return (S,a,nreactions,nvariables,ymat,Mumat, c)

def make_damat(amat, nMoments, ymat):

    """
    Calculate matrix of derivatives of rate equations ("damat")
    (n-1)th row gives nth order derivatives
    number of columns = nreactions
    each entry is a list with derivatives for that reaction/order

    In the end, damat contains the derivatives of all orders (ord), for all reactions (react),
    with respect to all species (sps):
    damat[ord][react][sps]
    All "mixed derivatives" are also calculated.

    :param amat: the propensity vector
    :param nMoments: the number of moments used in expansion
    :param ymat: the species vector

    :return: the "matrix" of derivation of amat
    """
    nreactions = len(amat)
    nvariables = len(ymat)

    nDerivatives = nMoments
    damat = Matrix(nDerivatives, 1, lambda i, j : 0)

    # At this point, `damat` is a matrix with 1 col and as many rows as moments. It is filled with 0s
    # `amat` is the column matrix of propensities (as many as reactions)
    # `ymat` is the column matrix of species (variables)

    # For all moment orders
    for D in range(0, nDerivatives):
        # if 0th order moment
        if D==0:
            # create an empty row
            row = []
            # For all reactions
            for na in range(0, nreactions):
                # create an empty vect of reactions
                reaction = []
                # for all variables/ all species
                for nv in range(0,nvariables):
                    # We derive the propensity of this reaction with respect to a species

                    deriv = diff(amat[na,0],ymat[nv,0])
                    reaction.append(deriv)
                # In the end, we get the partial derivatives of the propensity
                # of this reaction with respect to all species.
                row.append(reaction)
            # For all reactions in a given order of derivation D we have a row
            damat[D,0] = row
        else:
            # this does the same as above but does higher order derivatives from the results obtained before
            prev = Matrix(damat[D-1,0])
            row = []
            for na in range(0,nreactions):
                reaction = []
                prevna = prev[na,:]
                y = len(prevna)
                for eq in range(0,y):
                    for nv in range(0,nvariables):
                        deriv = diff(prevna[0,eq],ymat[nv,0])
                        reaction.append(deriv)
                row.append(reaction)
            damat[D,0] = row
    return damat

def make_T_matrix(nvariables, nreactions, TE_matrix, S):
    #TODO AFAIK, the variable T is unused. If this is true, this function is unnecessary
    T = []
    for nv in range(0,nvariables):
        row = []
        for nr in range(0,nreactions):
            Stmp = S[nv,nr]
            Ttmp = TE_matrix[nr,:]
            row.append(Stmp*Ttmp)
        T.append(row)
    return T

def substitute_mean_by_y(CentralMoments, nvariables):
    for i in range(0, nvariables):
        numv = [0] * nvariables
        numv[i] = 1
        numstr = str(numv[0])
        for j in range(1, nvariables):
            numstr = numstr + str(numv[j])
        t1 = Symbol('y_%d' %(i))
        t2 = Symbol('x'+numstr)

        for m in range(0, len(CentralMoments)):
            for n in range(0, len(CentralMoments[m])):
                CentralMoments[m][n] = Subs(CentralMoments[m][n], t2, t1).doit()
    return CentralMoments

def MFK_final(nMoments):

    time1 = time()
    output = open(str(sys.argv[3]),'w')

    # Define the kinetic model
    (S, amat, nreactions, nvariables, ymat, Mumat, c) = get_and_check_model()

    # Make the derivation matrix
    damat = make_damat(amat, nMoments, ymat)

    # compute counter and mcounter; the "k" and "n" vectors in equations. counter = mcounter - first_order_moments
    (counter, mcounter) = fcount(nMoments, nvariables)
    # Calculate TaylorExpansion terms to use in dmu/dt (eq. 6)
    TE_matrix = TaylorExpansion(nreactions,nvariables,damat,amat,counter,nMoments)

    # M is the product of the stoichiometry matrix by the Taylor Expansion terms.
    # one row per species and one col per element of counter
    M = S*TE_matrix

    # T seems unused, TODO
    T = make_T_matrix(nvariables, nreactions, TE_matrix, S)

    #  Calculate expressions to use in central moments equations (eq. 9)
    #  CentralMoments is a list with entry for each moment (n1,...,nd) combination.
    # TODO remove `nDerivatives`: it is simply an alias for nMoments
    nDerivatives = nMoments
    CentralMoments = eq_centralmoments(counter,mcounter,M,T,nvariables,ymat,nreactions,nMoments,amat,S,nDerivatives)

    #####################################################################
    #  Substitute means in CentralMoments by y_i (ymat entry)
    ####################################################################

    CentralMoments = substitute_mean_by_y(CentralMoments, nvariables)
    # print CentralMoments
    # for i in range(0, nvariables):
    #     numv = [0] * nvariables
    #     numv[i] = 1
    #     numstr = str(numv[0])
    #     for j in range(1, nvariables):
    #         numstr = numstr + str(numv[j])
    #     t1 = Symbol('y_%d' %(i))
    #     t2 = Symbol('x'+numstr)
    #
    #     for m in range(0, len(CentralMoments)):
    #         for n in range(0, len(CentralMoments[m])):
    #             CentralMoments[m][n] = Subs(CentralMoments[m][n], t2, t1).doit()

    #####################################################################
    #  Substitute higher order raw moments in terms of central moments
    #  raw_to_central calculates central moments (momvec) in terms
    #  of raw moment expressions (mom) (eq. 8)
    #####################################################################

    (mom, momvec) = raw_to_central(nvariables, counter, ymat, mcounter)

    # Substitute one for zeroth order raw moments in mom
    # Unnecessary outer loop
    for i in range(0, nvariables):
        numv = [0] * nvariables
        numstr = str(numv[0])
        for j in range(1, nvariables):
            numstr = numstr + str(numv[j])
        t1 = F(1)
        t2 = Symbol('x'+numstr)
        for m in range(0, len(mom)):
            mom[m] = Subs(mom[m], t2, t1).doit()

    # Substitute first order raw moments (means) in mom with y_i (ymat entry)
    for i in range(0, nvariables):
        numv = [0] * nvariables
        numv[i] = 1
        numstr = str(numv[0])
        for j in range(1, nvariables):
            numstr = numstr + str(numv[j])
        t1 = Symbol('y_%d' %(i))
        t2 = Symbol('x'+numstr)
        for m in range(0, len(mom)):
            mom[m] = Subs(mom[m], t2, t1).doit()

    # Substitute raw moment terms in CentralMoments in terms of central moments
    # (need to iterate in reverse from highest to lowest order moments to ensure all
    # raw moments are replaced as some higher order raw moments are expressed in terms
    # of central and lower order raw moments)

    for i in range(len(momvec)-1,-1,-1):
        string = str(momvec[i])
        num = string[2:]
        mm = Symbol('x'+num)
        soln = solve(mom[i]-momvec[i],mm)

        for m in range(0,len(CentralMoments)):
            for n in range(0,len(CentralMoments[m])):
                CentralMoments[m][n] = Subs(CentralMoments[m][n],mm, soln).doit()

    ##############################################################################
    # Substitute central moment terms ymn, where n gives n1,...nd combination
    # for yxi where i indicates index in counter for that n1,...,nd
    ##############################################################################

    if len(CentralMoments) != 0:
        nM = len(CentralMoments[0])
    else:
        nM = 1
    yms = Matrix(nM,1,lambda i,j:var('yx%d' % i))

    for i in range(0,len(momvec)):
        yx=Symbol('yx'+str(i+1))
        for m in range(0, len(CentralMoments)):
            for n in range(0, len(CentralMoments[m])):
                CentralMoments[m][n] = Subs(CentralMoments[m][n], momvec[i], yx).doit()
                try:
                    CentralMoments[m][n] = simplify(CentralMoments[m][n])
                except:
                    pass

    ##############################################################################
    # Get expressions for each central moment, and enter into list MFK
    ##############################################################################

    # Set zeroth order central moment to 1
    yms[0] = 1

    # Get expressions for higher order central moments
    MFK1 = M*yms
    MFK = []
    for i in range(0,len(MFK1)):
        try:
            MFK1[i] = simplify(MFK1[i])
        except:
            pass
        MFK.append(MFK1[i])

    for i in range(0,len(CentralMoments)):
        rowmat = Matrix(1,len(CentralMoments[i]),CentralMoments[i])
        MFK2 = rowmat*yms
        for j in range(0,len(MFK2)):
            try:
                MFK2[j] = simplify(MFK2[j])
            except:
                pass
            MFK.append(MFK2[j])

    ###############################################################################
    # Write information to output file (and moment names and equations to .tex file)
    ################################################################################

    # Create list with moment names (moment_list)

    moment_list = []
    for i in range(0, nvariables):
        means_vec = [0]*nvariables
        means_vec[i] = 1
        moment_list.append(means_vec)
    moment_list += counter[1:]

    # Get list of parameters for LHS of ODEs

    LHS = ymat[:]
    for moms in range(1,len(yms)):
        LHS.append(yms[moms])

    constants = c[:]

    out_tex = open(str(sys.argv[3])+'.tex','w')
    out_tex.write('\documentclass{article}\n\usepackage[landscape, margin=0.5in, a3paper]{geometry}\n\\begin{document}\n\section*{RHS of equations}\n')

    # Write equations, LHS, constants, moment names, and numbers of variables/moments/equations

    output.write('MEA\n\nRHS of equations:\n')
    for i in range(len(MFK)):
            output.write(str(MFK[i])+'\n')
            out_tex.write('$\dot '+str(latex(LHS[i]))+ ' = '+str(latex(MFK[i]))+'$\\\\\\\\')
    output.write('\nLHS:\n')
    out_tex.write('\n\section*{Moments}\n')
    for i in range(len(LHS)):
        output.write(str(LHS[i])+'\n')
        out_tex.write('\n$'+str(latex(LHS[i]))+'$: {'+str(moment_list[i])+'}\\\\')

    output.write('\nConstants:\n')
    for i in range(len(constants)):
        output.write(str(constants[i])+'\n')
    time2 = time()
    time3 = time2-time1
    output.write('Number of variables:\n'+str(nvariables)+'\nNumber of moments:\n'+str(nMoments)+'\nTime taken (s): '+str(time3))

    output.write('\n\nNumber of equations:\n'+str(len(LHS)))
    output.write('\n\nList of moments:\n')
    for i in range(len(moment_list)):
        output.write(str(moment_list[i])+'\n')

    output.close()

    out_tex.write('\n\end{document}')
    out_tex.close()

def get_args():
    model_ = sys.argv[1]
    numMoments = int(sys.argv[2])

    return (model_, numMoments)
if __name__ == "__main__":
    model_, numMoments = get_args()
    os.system('python formatmodel.py '+model_)
    MFK_final(numMoments)
<commit_msg>comments and restructuring in MEA.py<commit_after>####python MFK_final.py <model> <nMoments> <outputfile>

from time import time
import sys
import os

from sympy import Matrix, diff, Symbol, Subs, Eq, var, simplify
from fcount import fcount
from sympy import Matrix, diff, Symbol, Subs, Eq, var, simplify
from sympy import S as F
from sympy.solvers import solve
from TaylorExpansion import TaylorExpansion
from centralmoments import eq_centralmoments
from raw_to_central import raw_to_central
from sympy import latex

#############################################################################
#   MFK_final takes number of moments as input, and produces central
#   moment equations using the specified model
#############################################################################

def get_and_check_model():
    from model import model

    (S,a,nreactions,nvariables,ymat,Mumat, c) = model()

    #Delete temporary model file
    os.system('rm model.py*')
    #TODO use assertions instead of `if else`
    #Check stoichiometry matrix
    if S.cols==nreactions and S.rows==nvariables:
        print "S=okay"
    elif S.cols!=nreactions:
        print "Wrong number of reactions in S"
    elif S.rows!=nvariables:
        print "Wrong number of variables in S"

    return (S,a,nreactions,nvariables,ymat,Mumat, c)

def make_damat(amat, nMoments, ymat):

    """
    Calculate matrix of derivatives of rate equations ("damat")
    (n-1)th row gives nth order derivatives
    number of columns = nreactions
    each entry is a list with derivatives for that reaction/order

    In the end, damat contains the derivatives of all orders (ord), for all reactions (react),
    with respect to all species (sps):
    damat[ord][react][sps]
    All "mixed derivatives" are also calculated.

    :param amat: the propensity vector
    :param nMoments: the number of moments used in expansion
    :param ymat: the species vector

    :return: the "matrix" of derivation of amat
    """
    nreactions = len(amat)
    nvariables = len(ymat)

    nDerivatives = nMoments
    damat = Matrix(nDerivatives, 1, lambda i, j : 0)

    # At this point, `damat` is a matrix with 1 col and as many rows as moments. It is filled with 0s
    # `amat` is the column matrix of propensities (as many as reactions)
    # `ymat` is the column matrix of species (variables)

    # For all moment orders
    for D in range(0, nDerivatives):
        # if 0th order moment
        if D==0:
            # create an empty row
            row = []
            # For all reactions
            for na in range(0, nreactions):
                # create an empty vect of reactions
                reaction = []
                # for all variables/ all species
                for nv in range(0,nvariables):
                    # We derive the propensity of this reaction with respect to a species

                    deriv = diff(amat[na,0],ymat[nv,0])
                    reaction.append(deriv)
                # In the end, we get the partial derivatives of the propensity
                # of this reaction with respect to all species.
                row.append(reaction)
            # For all reactions in a given order of derivation D we have a row
            damat[D,0] = row
        else:
            # this does the same as above but does higher order derivatives from the results obtained before
            prev = Matrix(damat[D-1,0])
            row = []
            for na in range(0,nreactions):
                reaction = []
                prevna = prev[na,:]
                y = len(prevna)
                for eq in range(0,y):
                    for nv in range(0,nvariables):
                        deriv = diff(prevna[0,eq],ymat[nv,0])
                        reaction.append(deriv)
                row.append(reaction)
            damat[D,0] = row
    return damat

def make_T_matrix(nvariables, nreactions, TE_matrix, S):
    #TODO AFAIK, the variable T is unused. If this is true, this function is unnecessary
    T = []
    for nv in range(0,nvariables):
        row = []
        for nr in range(0,nreactions):
            Stmp = S[nv,nr]
            Ttmp = TE_matrix[nr,:]
            row.append(Stmp*Ttmp)
        T.append(row)
    return T

def substitute_mean_by_y(mom, nvariables):

    for i in range(0, nvariables):
        numv = [0] * nvariables
        numv[i] = 1
        numstr = str(numv[0])
        for j in range(1, nvariables):
            numstr = numstr + str(numv[j])
        t1 = Symbol('y_%d' %(i))
        t2 = Symbol('x'+numstr)

        for m in range(0, len(mom)):
            if isinstance(mom[m],list):
                for n in range(0, len(mom[m])):
                    mom[m][n] = Subs(mom[m][n], t2, t1).doit()
            else:
                mom[m] = Subs(mom[m], t2, t1).doit()

    return mom

def MFK_final(nMoments):

    time1 = time()
    output = open(str(sys.argv[3]),'w')

    # Define the kinetic model
    (S, amat, nreactions, nvariables, ymat, Mumat, c) = get_and_check_model()

    # Make the derivation matrix
    damat = make_damat(amat, nMoments, ymat)

    # compute counter and mcounter; the "k" and "n" vectors in equations. counter = mcounter - first_order_moments
    (counter, mcounter) = fcount(nMoments, nvariables)
    # Calculate TaylorExpansion terms to use in dmu/dt (eq. 6)
    TE_matrix = TaylorExpansion(nreactions,nvariables,damat,amat,counter,nMoments)

    # M is the product of the stoichiometry matrix by the Taylor Expansion terms.
    # one row per species and one col per element of counter
    M = S*TE_matrix

    # T seems unused, TODO
    T = make_T_matrix(nvariables, nreactions, TE_matrix, S)

    #  Calculate expressions to use in central moments equations (eq. 9)
    #  CentralMoments is a list with entry for each moment (n1,...,nd) combination.
    # TODO remove `nDerivatives`: it is simply an alias for nMoments
    nDerivatives = nMoments
    CentralMoments = eq_centralmoments(counter,mcounter, M,T,nvariables,ymat,nreactions,nMoments,amat,S,nDerivatives)

    #####################################################################
    #  Substitute means in CentralMoments by y_i (ymat entry)
    ####################################################################

    CentralMoments = substitute_mean_by_y(CentralMoments, nvariables)

    #####################################################################
    #  Substitute higher order raw moments in terms of central moments
    #  raw_to_central calculates central moments (momvec) in terms
    #  of raw moment expressions (mom) (eq. 8)
    #####################################################################

    (mom, momvec) = raw_to_central(nvariables, counter, ymat, mcounter)

    # Substitute one for zeroth order raw moments in mom
    # TODO unnecessary outer loop
    for i in range(0, nvariables):
        numv = [0] * nvariables
        numstr = str(numv[0])
        for j in range(1, nvariables):
            numstr = numstr + str(numv[j])

        t1 = F(1)
        t2 = Symbol('x'+numstr)
        for m in range(0, len(mom)):
            mom[m] = Subs(mom[m], t2, t1).doit()

    # Substitute first order raw moments (means) in mom with y_i (ymat entry)
    mom = substitute_mean_by_y(mom,nvariables)

    # Substitute raw moment terms in CentralMoments in terms of central moments
    # (need to iterate in reverse from highest to lowest order moments to ensure all
    # raw moments are replaced as some higher order raw moments are expressed in terms
    # of central and lower order raw moments)

    for i in range(len(momvec)-1,-1,-1):
        string = str(momvec[i])
        num = string[2:]
        mm = Symbol('x'+num)
        soln = solve(mom[i]-momvec[i],mm)

        for m in range(0,len(CentralMoments)):
            for n in range(0,len(CentralMoments[m])):
                tmp = (CentralMoments[m][n],mm, soln)
                CentralMoments[m][n] = Subs(CentralMoments[m][n],mm, soln).doit()
                if tmp != (CentralMoments[m][n],mm, soln):
                    print tmp
                    print (CentralMoments[m][n],mm, soln)

    ##############################################################################
    # Substitute central moment terms ymn, where n gives n1,...nd combination
    # for yxi where i indicates index in counter for that n1,...,nd
    ##############################################################################

    if len(CentralMoments) != 0:
        nM = len(CentralMoments[0])
    else:
        nM = 1
    yms = Matrix(nM,1,lambda i,j:var('yx%d' % i))

    for i in range(0,len(momvec)):
        yx = Symbol('yx'+str(i+1))
        print momvec[i]

        for m in range(0, len(CentralMoments)):
            for n in range(0, len(CentralMoments[m])):
                CentralMoments[m][n] = Subs(CentralMoments[m][n], momvec[i], yx).doit()
                try:
                    CentralMoments[m][n] = simplify(CentralMoments[m][n])
                except:
                    pass

    ##############################################################################
    # Get expressions for each central moment, and enter into list MFK
    ##############################################################################

    # Set zeroth order central moment to 1
    yms[0] = 1

    # Get expressions for higher order central moments
    MFK1 = M*yms
    MFK = []
    for i in range(0,len(MFK1)):
        try:
            MFK1[i] = simplify(MFK1[i])
        except:
            pass
        MFK.append(MFK1[i])

    for i in range(0,len(CentralMoments)):
        rowmat = Matrix(1,len(CentralMoments[i]),CentralMoments[i])
        MFK2 = rowmat*yms
        for j in range(0,len(MFK2)):
            try:
                MFK2[j] = simplify(MFK2[j])
            except:
                pass
            MFK.append(MFK2[j])

    ###############################################################################
    # Write information to output file (and moment names and equations to .tex file)
    ################################################################################

    # Create list with moment names (moment_list)

    moment_list = []
    for i in range(0, nvariables):
        means_vec = [0]*nvariables
        means_vec[i] = 1
        moment_list.append(means_vec)
    moment_list += counter[1:]

    # Get list of parameters for LHS of ODEs

    LHS = ymat[:]
    for moms in range(1,len(yms)):
        LHS.append(yms[moms])

    constants = c[:]

    out_tex = open(str(sys.argv[3])+'.tex','w')
    out_tex.write('\documentclass{article}\n\usepackage[landscape, margin=0.5in, a3paper]{geometry}\n\\begin{document}\n\section*{RHS of equations}\n')

    # Write equations, LHS, constants, moment names, and numbers of variables/moments/equations

    output.write('MEA\n\nRHS of equations:\n')
    for i in range(len(MFK)):
            output.write(str(MFK[i])+'\n')
            out_tex.write('$\dot '+str(latex(LHS[i]))+ ' = '+str(latex(MFK[i]))+'$\\\\\\\\')
    output.write('\nLHS:\n')
    out_tex.write('\n\section*{Moments}\n')
    for i in range(len(LHS)):
        output.write(str(LHS[i])+'\n')
        out_tex.write('\n$'+str(latex(LHS[i]))+'$: {'+str(moment_list[i])+'}\\\\')

    output.write('\nConstants:\n')
    for i in range(len(constants)):
        output.write(str(constants[i])+'\n')
    time2 = time()
    time3 = time2-time1
    output.write('Number of variables:\n'+str(nvariables)+'\nNumber of moments:\n'+str(nMoments)+'\nTime taken (s): '+str(time3))

    output.write('\n\nNumber of equations:\n'+str(len(LHS)))
    output.write('\n\nList of moments:\n')
    for i in range(len(moment_list)):
        output.write(str(moment_list[i])+'\n')

    output.close()

    out_tex.write('\n\end{document}')
    out_tex.close()

def get_args():
    model_ = sys.argv[1]
    numMoments = int(sys.argv[2])

    return (model_, numMoments)
if __name__ == "__main__":
    model_, numMoments = get_args()
    os.system('python formatmodel.py '+model_)
    MFK_final(numMoments)
<|endoftext|>


<commit_before>import os
import shutil
import re
import glob
import subprocess
import json
import sys
try:
    import pil
    import logger
    import helpers
    from constants import Constants
except ImportError:
    from . import pil
    from . import logger
    from . import helpers
    from .constants import Constants

"""
The content of this file was originally written by https://github.com/taengstagram
The code has been edited for use in PyInstaLive.
"""

def _get_file_index(filename):
    """ Extract the numbered index in filename for sorting """
    mobj = re.match(r'.+\-(?P<idx>[0-9]+)\.[a-z]+', filename)
    if mobj:
        return int(mobj.group('idx'))
    return -1

def assemble(user_called=True):
    try:
        ass_json_file = pil.assemble_arg if pil.assemble_arg.endswith(".json") else pil.assemble_arg + ".json"
        ass_mp4_file = os.path.join(pil.dl_path, os.path.basename(ass_json_file).replace("_downloads", "").replace(".json", ".mp4"))
        ass_segment_dir = pil.assemble_arg.split('.')[0]
        broadcast_info = {}
        if not os.path.isdir(ass_segment_dir) or not os.listdir(ass_segment_dir):
            logger.error('The segment directory does not exist or does not contain any files: %s' % ass_segment_dir)
            logger.separator()
            return
        if not os.path.isfile(ass_json_file):
            logger.warn("No matching json file found for the segment directory, trying to continue without it.")
            ass_stream_id = os.listdir(ass_segment_dir)[0].split('-')[0]
            broadcast_info['id'] = ass_stream_id
            broadcast_info['broadcast_status'] = "active"
            broadcast_info['segments'] = {}
        else:
            with open(ass_json_file) as info_file:
                broadcast_info = json.load(info_file)

        if broadcast_info.get('broadcast_status', '') == 'post_live':
            logger.error('Video segment files from replay downloads cannot be assembled.')
            return

        stream_id = str(broadcast_info['id'])

        segment_meta = broadcast_info.get('segments', {})
        if segment_meta:
            all_segments = [
                os.path.join(ass_segment_dir, k)
                for k in broadcast_info['segments'].keys()]
        else:
            all_segments = list(filter(
                os.path.isfile,
                glob.glob(os.path.join(ass_segment_dir, '%s-*.m4v' % stream_id))))

        all_segments = sorted(all_segments, key=lambda x: _get_file_index(x))
        sources = []
        audio_stream_format = 'assembled_source_{0}_{1}_mp4.tmp'
        video_stream_format = 'assembled_source_{0}_{1}_m4a.tmp'
        video_stream = ''
        audio_stream = ''

        if not all_segments:
            logger.error("No video segment files have been found in the specified folder.")
            logger.separator()
            return
        else:
            logger.info("Assembling video segment files from specified folder: {}".format(ass_segment_dir))

        for segment in all_segments:
            if not os.path.isfile(segment.replace('.m4v', '.m4a')):
                logger.warn('Audio segment not found: {0!s}'.format(segment.replace('.m4v', '.m4a')))
                continue

            if segment.endswith('-init.m4v'):
                logger.info('Replacing %s' % segment)
                segment = os.path.join(
                    os.path.dirname(os.path.realpath(__file__)), 'repair', 'init.m4v')

            if segment.endswith('-0.m4v'):
                continue

            video_stream = os.path.join(
                ass_segment_dir, video_stream_format.format(stream_id, len(sources)))
            audio_stream = os.path.join(
                ass_segment_dir, audio_stream_format.format(stream_id, len(sources)))

            file_mode = 'ab'

            with open(video_stream, file_mode) as outfile, open(segment, 'rb') as readfile:
                shutil.copyfileobj(readfile, outfile)

            with open(audio_stream, file_mode) as outfile, open(segment.replace('.m4v', '.m4a'), 'rb') as readfile:
                shutil.copyfileobj(readfile, outfile)

        if audio_stream and video_stream:
            sources.append({'video': video_stream, 'audio': audio_stream})

        for n, source in enumerate(sources):
            ffmpeg_binary = os.getenv('FFMPEG_BINARY', 'ffmpeg')
            cmd = [
                ffmpeg_binary, '-loglevel', 'warning', '-y',
                '-i', source['audio'],
                '-i', source['video'],
                '-c:v', 'copy', '-c:a', 'copy', ass_mp4_file]
            #fnull = open(os.devnull, 'w')
            fnull = None
            exit_code = subprocess.call(cmd, stdout=fnull, stderr=subprocess.STDOUT)
            if exit_code != 0:
                logger.warn("FFmpeg exit code not '0' but '{:d}'.".format(exit_code))
            else:
                logger.info('The video file has been generated: %s' % os.path.basename(ass_mp4_file))
            if user_called:
                logger.separator()
    except Exception as e:
        logger.error("An error occurred: {:s}".format(str(e)))<commit_msg>Retry assemble including -0 segment if first attempt fails<commit_after>import os
import shutil
import re
import glob
import subprocess
import json
import sys
try:
    import pil
    import logger
    import helpers
    from constants import Constants
except ImportError:
    from . import pil
    from . import logger
    from . import helpers
    from .constants import Constants

"""
The content of this file was originally written by https://github.com/taengstagram
The code has been edited for use in PyInstaLive.
"""

def _get_file_index(filename):
    """ Extract the numbered index in filename for sorting """
    mobj = re.match(r'.+\-(?P<idx>[0-9]+)\.[a-z]+', filename)
    if mobj:
        return int(mobj.group('idx'))
    return -1

def assemble(user_called=True, retry_with_zero_m4v=False):
    try:
        ass_json_file = pil.assemble_arg if pil.assemble_arg.endswith(".json") else pil.assemble_arg + ".json"
        ass_mp4_file = os.path.join(pil.dl_path, os.path.basename(ass_json_file).replace("_downloads", "").replace(".json", ".mp4"))
        ass_segment_dir = pil.assemble_arg.split('.')[0]
        broadcast_info = {}
        if not os.path.isdir(ass_segment_dir) or not os.listdir(ass_segment_dir):
            logger.error('The segment directory does not exist or does not contain any files: %s' % ass_segment_dir)
            logger.separator()
            return
        if not os.path.isfile(ass_json_file):
            logger.warn("No matching json file found for the segment directory, trying to continue without it.")
            ass_stream_id = os.listdir(ass_segment_dir)[0].split('-')[0]
            broadcast_info['id'] = ass_stream_id
            broadcast_info['broadcast_status'] = "active"
            broadcast_info['segments'] = {}
        else:
            with open(ass_json_file) as info_file:
                broadcast_info = json.load(info_file)

        if broadcast_info.get('broadcast_status', '') == 'post_live':
            logger.error('Video segment files from replay downloads cannot be assembled.')
            return

        stream_id = str(broadcast_info['id'])

        segment_meta = broadcast_info.get('segments', {})
        if segment_meta:
            all_segments = [
                os.path.join(ass_segment_dir, k)
                for k in broadcast_info['segments'].keys()]
        else:
            all_segments = list(filter(
                os.path.isfile,
                glob.glob(os.path.join(ass_segment_dir, '%s-*.m4v' % stream_id))))

        all_segments = sorted(all_segments, key=lambda x: _get_file_index(x))
        sources = []
        audio_stream_format = 'assembled_source_{0}_{1}_mp4.tmp'
        video_stream_format = 'assembled_source_{0}_{1}_m4a.tmp'
        video_stream = ''
        audio_stream = ''
        has_skipped_zero_m4v = False

        if not all_segments:
            logger.error("No video segment files have been found in the specified folder.")
            logger.separator()
            return
        else:
            logger.info("Assembling video segment files from specified folder: {}".format(ass_segment_dir))

        for segment in all_segments:
            if not os.path.isfile(segment.replace('.m4v', '.m4a')):
                logger.warn('Audio segment not found: {0!s}'.format(segment.replace('.m4v', '.m4a')))
                continue

            if segment.endswith('-init.m4v'):
                logger.info('Replacing %s' % segment)
                segment = os.path.join(
                    os.path.dirname(os.path.realpath(__file__)), 'repair', 'init.m4v')

            if segment.endswith('-0.m4v') and not retry_with_zero_m4v:
                has_skipped_zero_m4v = True
                continue

            video_stream = os.path.join(
                ass_segment_dir, video_stream_format.format(stream_id, len(sources)))
            audio_stream = os.path.join(
                ass_segment_dir, audio_stream_format.format(stream_id, len(sources)))

            file_mode = 'ab'

            with open(video_stream, file_mode) as outfile, open(segment, 'rb') as readfile:
                shutil.copyfileobj(readfile, outfile)

            with open(audio_stream, file_mode) as outfile, open(segment.replace('.m4v', '.m4a'), 'rb') as readfile:
                shutil.copyfileobj(readfile, outfile)

        if audio_stream and video_stream:
            sources.append({'video': video_stream, 'audio': audio_stream})

        for n, source in enumerate(sources):
            ffmpeg_binary = os.getenv('FFMPEG_BINARY', 'ffmpeg')
            cmd = [
                ffmpeg_binary, '-loglevel', 'warning', '-y',
                '-i', source['audio'],
                '-i', source['video'],
                '-c:v', 'copy', '-c:a', 'copy', ass_mp4_file]
            #fnull = open(os.devnull, 'w')
            fnull = None
            exit_code = subprocess.call(cmd, stdout=fnull, stderr=subprocess.STDOUT)
            if exit_code != 0:
                logger.warn("FFmpeg exit code not '0' but '{:d}'.".format(exit_code))
                if has_skipped_zero_m4v and not retry_with_zero_m4v:
                    logger.binfo("*-0.m4v segment was detected but skipped, retrying to assemble video without "
                                 "skipping it.")
                    os.remove(source['audio'])
                    os.remove(source['video'])
                    logger.separator()
                    assemble(user_called, retry_with_zero_m4v=True)
                    return
            else:
                logger.info('The video file has been generated: %s' % os.path.basename(ass_mp4_file))
            if user_called:
                logger.separator()
    except Exception as e:
        logger.error("An error occurred: {:s}".format(str(e)))<|endoftext|>