#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai

# Changelog
#  0.11 - Version by adamselene
#  0.11pd - Tweaked version by pdurrant
#  0.12 - extracts pictures too, and all into a folder.
#  0.13 - added back in optional output dir for those who don't want it based on infile
#  0.14 - auto flush stdout and wrapped in main, added proper return codes
#  0.15 - added support for metadata
#  0.16 - metadata now starting to be output as an opf file (PD)
#  0.17 - Also created tweaked text as source for Mobipocket Creator
#  0.18 - removed raw mobi file completely but kept _meta.html file for ease of conversion
#  0.19 - added in metadata for ASIN, Updated Title and Rights to the opf
#  0.20 - remove _meta.html since no longer needed
#  0.21 - Fixed some typos in the opf output, and also updated handling
#         of test for trailing data/multibyte characters
#  0.22 - Fixed problem with > 9 images
#  0.23 - Now output Start guide item
#  0.24 - Set firstaddl value for 'TEXtREAd'
#  0.25 - Now added character set metadata to html file for utf-8 files.
#  0.26 - Dictionary support added. Image handling speed improved.
#         For huge files create temp files to speed up decoding.
#         Language decoding fixed. Metadata is now converted to utf-8 when written to opf file.
#  0.27 - Add idx:entry attribute "scriptable" if dictionary contains entry length tags.
#         Don't save non-image sections as images. Extract and save source zip file
#         included by kindlegen as kindlegensrc.zip.
#  0.28 - Added back correct image file name extensions, created FastConcat class to simplify and clean up
#  0.29 - Metadata handling reworked, multiple entries of the same type are now supported.
#         Several missing types added.
#         FastConcat class has been removed as in-memory handling with lists is faster, even for huge files.
#  0.30 - Add support for outputting **all** metadata values - encode content with hex if of unknown type
#  0.31 - Now supports Print Replica ebooks, outputting PDF and mysterious data sections
#  0.32 - Now supports NCX file extraction/building.
#                 Overhauled the structure of mobiunpack to be more class oriented.
#  0.33 - Split Classes ito separate files and added prelim support for KF8 format eBooks
#  0.34 - Improved KF8 support, guide support, bug fixes
#  0.35 - Added splitting combo mobi7/mobi8 into standalone mobi7 and mobi8 files
#         Also handle mobi8-only file properly
#  0.36 - very minor changes to support KF8 mobis with no flow items, no ncx, etc
#  0.37 - separate output, add command line switches to control, interface to Mobi_Unpack.pyw
#  0.38 - improve split function by resetting flags properly, fix bug in Thumbnail Images
#  0.39 - improve split function so that ToC info is not lost for standalone mobi8s
#  0.40 - make mobi7 split match official versions, add support for graphic novel metadata,
#         improve debug for KF8
#  0.41 - fix when StartOffset set to 0xffffffff, fix to work with older mobi versions,
#         fix other minor metadata issues
#  0.42 - add new class interface to allow it to integrate more easily with internal calibre routines
#  0.43 - bug fixes for new class interface
#  0.44 - more bug fixes and fix for potnetial bug caused by not properly closing created zip archive
#  0.45 - sync to version in the new Mobi_Unpack plugin
#  0.46 - fixes for: obfuscated fonts, improper toc links and ncx, add support for opentype fonts
#  0.47 - minor opf improvements
#  0.48 - ncx link fixes
#  0.49 - use azw3 when splitting mobis
#  0.50 - unknown change
#  0.51 - fix for converting filepos links to hrefs, Added GPL3 notice, made KF8 extension just '.azw3'
#  0.52 - fix for cover metadata (no support for Mobipocket Creator)
#  0.53 - fix for proper identification of embedded fonts, added new metadata items
#  0.54 - Added error-handling so wonky embedded fonts don't bomb the whole unpack process,
#         entity escape KF8 metadata to ensure valid OPF.
#  0.55  Strip extra StartOffset EXTH from the mobi8 header when splitting, keeping only the relevant one
#         For mobi8 files, don't generate duplicate guide entries from the metadata if we could extract one
#         from the OTH table.
#  0.56 - Added further entity escaping of OPF text.
#         Allow unicode string file paths to be passed as arguments to the unpackBook method without blowing up later
#         when the attempt to "re"-unicode a portion of that filename occurs in the process_all_mobi_headers method.
#  0.57 - Fixed eror when splitting Preview files downloaded from KDP website
#  0.58 - Output original kindlegen build log ('CMET' record) if included in the package.
#  0.58 - Include and extend functionality of DumpMobiHeader, replacing DEBUG with DUMP
#  0.59 - Much added DUMP functionality, including full dumping and descriptions of sections
#  0.60 - Bug fixes in opf, div tables, bad links, page breaks, section descriptions
#       - plus a number of other bug fixed that were found by Sergey Dubinets
#       - fixs for file/paths that require full unicode to work properly
#       - replace subprocess with multiprocessing to remove need for unbuffered stdout
#  0.61 - renamed to be KindleUnpack and more unicode/utf-8 path bug fixes and other minor fixes
#  0.62 - fix for multiprocessing on Windows, split fixes, opf improvements
#  0.63 - Modified to process right to left page progression books properly.
#       - Added some id_map_strings and RESC section processing; metadata and
#       - spine in the RESC are integrated partly to content.opf.
#  0.63a- Separated K8 RESC processor to an individual file. Bug fixes. Added cover page creation.
#  0.64 - minor bug fixes to more properly handle unicode command lines, and support for more jpeg types
#  0.64a- Modifed to handle something irregular mobi and azw3 files.
#  0.64b- Modifed to create k8resc.spine for no RECS files.
#  0.65 - Bug fixes to shorten title and remove epub3 "properties" to make the output epub2 compliant
#  0.65a- Bug fixes to extract RESC section correctly, to prevent item id confliction
#       - and to process multiline comments in RESC.
#  0.66 - Bug fix to deal with missing first resource information sometimes generated by calibre
#  0.66a- Fixed minor bugs, which probably do not affect the output anything
#  0.67 - Fixed Mobi Split functionality bug with azw3 images not being properly copied
#  0.68 - preliminary support for handling PAGE sections to create page-map.xml
#  0.69 - preliminary support for CONT and CRES for HD Images
#  0.70 - preliminary support for decoding apnx files when used with azw3 ebooks
#  0.71 - extensive refactoring of kindleunpack.py to make it more manageable
#  0.72 - many bug fixes from tkeo: fix pageProcessing, fix print replica, fix resc usage, fix font mangling, etc.
#  0.72a- fix for still broken PrintReplica support
#  0.72b- preview for primary epub3 support. A parameter epubver(default='2') is added to process_all_mobi_headers(), unpackBook().
#  0.72c- preview for apnx page support
#  0.72d- more bugs fixed in preview features, much improved GUI with ability to dynaically grow the Log Window with preference support
#  0.72e- more bug fixes, Tk GUI adds support for epub version and HDImage use
#  0.72f- more bug fixes, implement use hd images if present
#  0.72g- minor bug fixes and cleanups from tkeo
#  0.72h- updated mobi_header and mobi_k8proc to use the correct fragment and guide terms in place of div and other
#         to better match the terms that both Calibre and Amazon use internally to their own software
#  0.72x- very experimental conversion to use new mobi_k8resc.py and some of its associated changes
#  0.72y- more changes to simplify and integrate in epub3 support in a simpler manner
#  0.72z- remove redundancy in mobi_opf.py and bug fixes for mobi_k8resc.py
#  0.73   faster mobi split, numerous bug fixes in mobi_k8proc, mobi_header, mobi_opf, mobi_k8resc, etc
#  0.74   added refines metadata, fixed language code in ncx and title in nav, added support for opf: from refines
#  0.75   much improved dictioanry support including support for multiple inflection sections, minor mobi_opf fixes


DUMP = False
""" Set to True to dump all possible information. """

WRITE_RAW_DATA = False
""" Set to True to create additional files with raw data for debugging/reverse engineering. """

SPLIT_COMBO_MOBIS = False
""" Set to True to split combination mobis into mobi7 and mobi8 pieces. """

CREATE_COVER_PAGE = True # XXX experimental
""" Create and insert a cover xhtml page. """

EOF_RECORD = chr(0xe9) + chr(0x8e) + "\r\n"
""" The EOF record content. """

TERMINATION_INDICATOR1 = chr(0x0)
TERMINATION_INDICATOR2 = chr(0x0) + chr(0x0)
TERMINATION_INDICATOR3 = chr(0x0) + chr(0x0) + chr(0x0)

KINDLEGENSRC_FILENAME = "kindlegensrc.zip"
""" The name for the kindlegen source archive. """

KINDLEGENLOG_FILENAME = "kindlegenbuild.log"
""" The name for the kindlegen build log. """

K8_BOUNDARY = "BOUNDARY"
""" The section data that divides K8 mobi ebooks. """


import sys
import os
import locale
import codecs

from utf8_utils import utf8_argv, add_cp65001_codec, utf8_str
add_cp65001_codec()
from path import pathof


import array, struct, re, imghdr, zlib, zipfile, datetime
import getopt, binascii
import uuid

class unpackException(Exception):
    pass


# import the kindleunpack support libraries
from unpack_structure import fileNames
from mobi_sectioner import Sectionizer, describe
from mobi_header import MobiHeader, dump_contexth
from mobi_utils import getLanguage, toHex, fromBase32, toBase32, mangle_fonts
from mobi_opf import OPFProcessor
from mobi_html import HTMLProcessor, XHTMLK8Processor
from mobi_ncx import ncxExtract
from mobi_k8proc import K8Processor
from mobi_split import mobi_split
from mobi_k8resc import K8RESCProcessor
from mobi_nav import NAVProcessor
from mobi_cover import CoverProcessor, get_image_type
from mobi_pagemap import PageMapProcessor
from mobi_dict import dictSupport


def processSRCS(i, files, imgnames, sect, data):
    # extract the source zip archive and save it.
    print "File contains kindlegen source archive, extracting as %s" % KINDLEGENSRC_FILENAME
    srcname = os.path.join(files.outdir, KINDLEGENSRC_FILENAME)
    open(pathof(srcname), 'wb').write(data[16:])
    imgnames.append(None)
    sect.setsectiondescription(i,"Zipped Source Files")
    return imgnames


def processPAGE(i, files, imgnames, sect, data, mh, pagemapproc):
    # process any page map information and create an apnx file
    pagemapproc = PageMapProcessor(mh, data)
    imgnames.append(None)
    sect.setsectiondescription(i,"PageMap")
    apnx_meta = {}
    apnx_meta['acr'] = str(sect.palmname).rstrip('\x00')
    apnx_meta['cdeType'] = mh.metadata['cdeType'][0]
    apnx_meta['contentGuid'] = hex(int(mh.metadata['UniqueID'][0]))[2:]
    apnx_meta['asin'] = mh.metadata['ASIN'][0]
    apnx_meta['pageMap'] = pagemapproc.getPageMap()
    if mh.version == 8:
        apnx_meta['format'] = 'MOBI_8'
    else:
        apnx_meta['format'] = 'MOBI_7'
    apnx_data = pagemapproc.generateAPNX(apnx_meta)
    if mh.isK8():
        outname = os.path.join(files.outdir, 'mobi8-'+files.getInputFileBasename() + '.apnx')
    else:
        outname = os.path.join(files.outdir, 'mobi7-'+files.getInputFileBasename() + '.apnx')
    open(pathof(outname), 'wb').write(apnx_data)
    return imgnames, pagemapproc


def processCMET(i, files, imgnames, sect, data):
    # extract the build log
    print "File contains kindlegen build log, extracting as %s" % KINDLEGENLOG_FILENAME
    srcname = os.path.join(files.outdir, KINDLEGENLOG_FILENAME)
    open(pathof(srcname), 'wb').write(data[10:])
    imgnames.append(None)
    sect.setsectiondescription(i,"Kindlegen log")
    return imgnames


# fonts only exist in KF8 ebooks
# Format:  bytes  0 -  3:  'FONT'
#          bytes  4 -  7:  uncompressed size
#          bytes  8 - 11:  flags
#              flag bit 0x0001 - zlib compression
#              flag bit 0x0002 - obfuscated with xor string
#          bytes 12 - 15:  offset to start of compressed font data
#          bytes 16 - 19:  length of xor string stored before the start of the comnpress font data
#          bytes 20 - 23:  start of xor string
def processFONT(i, files, imgnames, sect, data, obfuscate_data):
    fontname = "font%05d" % i
    ext = '.dat'
    font_error = False
    font_data = data
    try:
        usize, fflags, dstart, xor_len, xor_start = struct.unpack_from('>LLLLL',data,4)
    except:
        print "Failed to extract font: {0:s} from section {1:d}".format(fontname,i)
        font_error = True
        ext = '.failed'
        pass
    if not font_error:
        print "Extracting font:", fontname
        font_data = data[dstart:]
        extent = len(font_data)
        extent = min(extent, 1040)
        if fflags & 0x0002:
            # obfuscated so need to de-obfuscate the first 1040 bytes
            key = bytearray(data[xor_start: xor_start+ xor_len])
            buf = bytearray(font_data)
            for n in xrange(extent):
                buf[n] ^=  key[n%xor_len]
            font_data = bytes(buf)
        if fflags & 0x0001:
            # ZLIB compressed data
            font_data = zlib.decompress(font_data)
        hdr = font_data[0:4]
        if hdr == '\0\1\0\0' or hdr == 'true' or hdr == 'ttcf':
            ext = '.ttf'
        elif hdr == 'OTTO':
            ext = '.otf'
        else:
            print "Warning: unknown font header %s" % hdr.encode('hex')
        if (ext == '.ttf' or ext == '.otf') and (fflags & 0x0002):
            obfuscate_data.append(fontname + ext)
        fontname += ext
        outfnt = os.path.join(files.imgdir, fontname)
        open(pathof(outfnt), 'wb').write(font_data)
        imgnames.append(fontname)
        sect.setsectiondescription(i,"Font {0:s}".format(fontname))
    return imgnames, obfuscate_data


def processCRES(i, files, imgnames, sect, data, beg, image_ptr, use_hd):
    # extract an HDImage
    global DUMP
    data = data[12:]
    imgtype = get_image_type(None, data)

    if imgtype is None:
        print "Warning: CRES Section %s does not contain a recognised resource" % i
        imgnames.append(None)
        sect.setsectiondescription(i,"Mysterious CRES data, first four bytes %s" % describe(data[0:4]))
        if DUMP:
            fname = "unknown%05d.dat" % i
            outname= os.path.join(files.outdir, fname)
            open(pathof(outname), 'wb').write(data)
            sect.setsectiondescription(i,"Mysterious CRES data, first four bytes %s extracting as %s" % (describe(data[0:4]), fname))
        image_ptr += 1
        return imgnames, image_ptr

    if use_hd:
        # overwrite corresponding lower res image with hd version
        imgname = imgnames[image_ptr]
        imgdest = files.imgdir
    else:
        imgname = "HDimage%05d.%s" % (i, imgtype)
        imgdest = files.hdimgdir
    print "Extracting HD image: {0:s} from section {1:d}".format(imgname,i)
    outimg = os.path.join(imgdest, imgname)
    open(pathof(outimg), 'wb').write(data)
    imgnames.append(None)
    sect.setsectiondescription(i,"Optional HD Image {0:s}".format(imgname))
    image_ptr += 1
    return imgnames, image_ptr


def processCONT(i, files, imgnames, sect, data):
    global DUMP
    # process a container header, most of this is unknown
    # right now only extract its EXTH
    dt = data[0:12]
    if dt == "CONTBOUNDARY":
        imgnames.append(None)
        sect.setsectiondescription(i,"CONTAINER BOUNDARY")
    else:
        sect.setsectiondescription(i,"CONT Header")
        imgnames.append(None)
        if DUMP:
            cpage, = struct.unpack_from('>L', data, 12)
            contexth = data[48:]
            print "\n\nContainer EXTH Dump"
            dump_contexth(cpage, contexth)
            fname = "CONT_Header%05d.dat" % i
            outname= os.path.join(files.outdir, fname)
            open(pathof(outname), 'wb').write(data)
    return imgnames


def processkind(i, files, imgnames, sect, data):
    global DUMP
    dt = data[0:12]
    if dt == "kindle:embed":
        if DUMP:
            print "\n\nHD Image Container Description String"
            print data
        sect.setsectiondescription(i,"HD Image Container Description String")
        imgnames.append(None)
    return imgnames


# spine information from the original content.opf
def processRESC(i, files, imgnames, sect, data, k8resc):
    global DUMP
    if DUMP:
        rescname = "RESC%05d.dat" % i
        print "Extracting Resource: ", rescname
        outrsc = os.path.join(files.outdir, rescname)
        open(pathof(outrsc), 'wb').write(data)
    if True: #try:
        # parse the spine and metadata from RESC
        k8resc = K8RESCProcessor(data[16:], DUMP)
    else: # except:
        print "Warning: cannot extract information from RESC."
        k8resc = None
    imgnames.append(None)
    sect.setsectiondescription(i,"K8 RESC section")
    return imgnames, k8resc


def processImage(i, files, imgnames, sect, data, beg, image_ptr, cover_offset):
    global DUMP
    # Extract an Image
    imgtype = get_image_type(None, data)
    if imgtype is None:
        print "Warning: Section %s does not contain a recognised resource" % i
        imgnames.append(None)
        sect.setsectiondescription(i,"Mysterious Section, first four bytes %s" % describe(data[0:4]))
        if DUMP:
            fname = "unknown%05d.dat" % i
            outname= os.path.join(files.outdir, fname)
            open(pathof(outname), 'wb').write(data)
            sect.setsectiondescription(i,"Mysterious Section, first four bytes %s extracting as %s" % (describe(data[0:4]), fname))
        return imgnames, image_ptr

    imgname = "image%05d.%s" % (i, imgtype)
    if cover_offset is not None and i == beg + cover_offset:
        imgname = "cover%05d.%s" % (i, imgtype)
    print "Extracting image: {0:s} from section {1:d}".format(imgname,i)
    outimg = os.path.join(files.imgdir, imgname)
    open(pathof(outimg), 'wb').write(data)
    imgnames.append(imgname)
    sect.setsectiondescription(i,"Image {0:s}".format(imgname))
    if image_ptr == -1:
        image_ptr = i - beg
    return imgnames, image_ptr


def processPrintReplica(metadata, files, imgnames, mh):
    global DUMP
    global WRITE_RAW_DATA
    rawML = mh.getRawML()
    if DUMP or WRITE_RAW_DATA:
        outraw = os.path.join(files.outdir,files.getInputFileBasename() + '.rawpr')
        open(pathof(outraw),'wb').write(rawML)

    fileinfo = []
    print "Print Replica ebook detected"
    try:
        numTables, = struct.unpack_from('>L', rawML, 0x04)
        tableIndexOffset = 8 + 4*numTables
        # for each table, read in count of sections, assume first section is a PDF
        # and output other sections as binary files
        paths = []
        for i in xrange(numTables):
            sectionCount, = struct.unpack_from('>L', rawML, 0x08 + 4*i)
            for j in xrange(sectionCount):
                sectionOffset, sectionLength, = struct.unpack_from('>LL', rawML, tableIndexOffset)
                tableIndexOffset += 8
                if j == 0:
                    entryName = os.path.join(files.outdir, files.getInputFileBasename() + ('.%03d.pdf' % (i+1)))
                else:
                    entryName = os.path.join(files.outdir, files.getInputFileBasename() + ('.%03d.%03d.data' % ((i+1),j)))
                open(pathof(entryName), 'wb').write(rawML[sectionOffset:(sectionOffset+sectionLength)])
    except Exception, e:
        print 'Error processing Print Replica: ' + str(e)

    fileinfo.append([None,'', files.getInputFileBasename() + '.pdf'])
    usedmap = {}
    for name in imgnames:
        if name is not None:
            usedmap[name] = 'used'
    opf = OPFProcessor(files, metadata, fileinfo, imgnames, False, mh, usedmap)
    opf.writeOPF()



def processMobi8(mh, metadata, sect, files, imgnames, pagemapproc, k8resc, obfuscate_data, apnxfile=None, epubver='2'):
    global DUMP
    global WRITE_RAW_DATA

    # extract raw markup langauge
    rawML = mh.getRawML()
    if DUMP or WRITE_RAW_DATA:
        outraw = os.path.join(files.k8dir,files.getInputFileBasename() + '.rawml')
        open(pathof(outraw),'wb').write(rawML)

    # KF8 require other indexes which contain parsing information and the FDST info
    # to process the rawml back into the xhtml files, css files, svg image files, etc
    k8proc = K8Processor(mh, sect, files, DUMP)
    k8proc.buildParts(rawML)

    # collect information for the guide first
    guidetext = k8proc.getGuideText()

    # if the guide was empty, add in any guide info from metadata, such as StartOffset
    if not guidetext and 'StartOffset' in metadata.keys():
        # Apparently, KG 2.5 carries over the StartOffset from the mobi7 part...
        # Taking that into account, we only care about the *last* StartOffset, which
        # should always be the correct one in these cases (the one actually pointing
        # to the right place in the mobi8 part).
        starts = metadata['StartOffset']
        last_start = starts[-1]
        last_start = int(last_start)
        if last_start == 0xffffffff:
            last_start = 0
        seq, idtext = k8proc.getFragTblInfo(last_start)
        filename, idtext = k8proc.getIDTagByPosFid(toBase32(seq), '0000000000')
        linktgt = filename
        if idtext != '':
            linktgt += '#' + idtext
        guidetext += '<reference type="text" href="Text/%s" />\n' % linktgt

    # if apnxfile is passed in use it for page map information
    if apnxfile is not None and pagemapproc is None:
        apnxdata = "00000000" + file(apnxfile, 'rb').read()
        pagemapproc = PageMapProcessor(mh, apnxdata)

    # generate the page map
    pagemapxml = ''
    if pagemapproc is not None:
        pagemapxml = pagemapproc.generateKF8PageMapXML(k8proc)
        outpm = os.path.join(files.k8oebps,'page-map.xml')
        open(pathof(outpm),'wb').write(pagemapxml)
        if DUMP:
            print pagemapproc.getNames()
            print pagemapproc.getOffsets()
            print "\n\nPage Map"
            print pagemapxml

    # process the toc ncx
    # ncx map keys: name, pos, len, noffs, text, hlvl, kind, pos_fid, parent, child1, childn, num
    print "Processing ncx / toc"
    ncx = ncxExtract(mh, files)
    ncx_data = ncx.parseNCX()
    # extend the ncx data with filenames and proper internal idtags
    for i in range(len(ncx_data)):
        ncxmap = ncx_data[i]
        [junk1, junk2, junk3, fid, junk4, off] = ncxmap['pos_fid'].split(':')
        filename, idtag = k8proc.getIDTagByPosFid(fid, off)
        ncxmap['filename'] = filename
        ncxmap['idtag'] = idtag
        ncx_data[i] = ncxmap

    # convert the rawML to a set of xhtml files
    print "Building an epub-like structure"
    htmlproc = XHTMLK8Processor(imgnames, k8proc)
    usedmap = htmlproc.buildXHTML()


    # write out the xhtml svg, and css files
    # fileinfo = [skelid|coverpage, dir, name]
    fileinfo = []
    # first create a cover page if none exists
    if CREATE_COVER_PAGE:
        cover = CoverProcessor(files, metadata, imgnames)
        cover_img = cover.getImageName()
        need_to_create_cover_page = False
        if cover_img is not None:
            if k8resc is None or not k8resc.hasSpine():
                part = k8proc.getPart(0)
                if part.find(cover_img) == -1:
                    need_to_create_cover_page = True
            else:
                if "coverpage" not in k8resc.spine_idrefs.keys():
                    part = k8proc.getPart(int(k8resc.spine_order[0]))
                    if part.find(cover_img) == -1:
                        k8resc.prepend_to_spine("coverpage", "inserted", "no", None)
                if k8resc.spine_order[0] == "coverpage":
                    need_to_create_cover_page = True
            if need_to_create_cover_page:
                filename = cover.getXHTMLName()
                fileinfo.append(["coverpage", 'Text', filename])
                guidetext += cover.guide_toxml()
                cover.writeXHTML()

    n =  k8proc.getNumberOfParts()
    for i in range(n):
        part = k8proc.getPart(i)
        [skelnum, dir, filename, beg, end, aidtext] = k8proc.getPartInfo(i)
        fileinfo.append([str(skelnum), dir, filename])
        fname = os.path.join(files.k8oebps,dir,filename)
        open(pathof(fname),'wb').write(part)
    n = k8proc.getNumberOfFlows()
    for i in range(1, n):
        [type, format, dir, filename] = k8proc.getFlowInfo(i)
        flowpart = k8proc.getFlow(i)
        if format == 'file':
            fileinfo.append([None, dir, filename])
            fname = os.path.join(files.k8oebps,dir,filename)
            open(pathof(fname),'wb').write(flowpart)

    # create the opf
    opf = OPFProcessor(files, metadata.copy(), fileinfo, imgnames, True, mh, usedmap, pagemapxml, guidetext, k8resc, epubver)
    uuid = opf.writeOPF(bool(obfuscate_data))

    if opf.hasNCX():
        # Create a toc.ncx.
        ncx.writeK8NCX(ncx_data, metadata)
    if opf.hasNAV():
        # Create a navigation document.
        nav = NAVProcessor(files)
        nav.writeNAV(ncx_data, guidetext, metadata)

    # make an epub-like structure of it all
    print "Creating an epub-like file"
    files.makeEPUB(usedmap, obfuscate_data, uuid)


def processMobi7(mh, metadata, sect, files, imgnames):
    global DUMP
    global WRITE_RAW_DATA
    # An original Mobi
    rawML = mh.getRawML()
    if DUMP or WRITE_RAW_DATA:
        outraw = os.path.join(files.mobi7dir,files.getInputFileBasename() + '.rawml')
        open(pathof(outraw),'wb').write(rawML)

    # process the toc ncx
    # ncx map keys: name, pos, len, noffs, text, hlvl, kind, pos_fid, parent, child1, childn, num
    ncx = ncxExtract(mh, files)
    ncx_data = ncx.parseNCX()
    ncx.writeNCX(metadata)

    positionMap = {}

    # if Dictionary build up the positionMap
    if mh.isDictionary():
        if mh.DictInLanguage():
            metadata['DictInLanguage'] = [mh.DictInLanguage()]
        if mh.DictOutLanguage():
            metadata['DictOutLanguage'] = [mh.DictOutLanguage()]
        positionMap = dictSupport(mh, sect).getPositionMap()

    # convert the rawml back to Mobi ml
    proc = HTMLProcessor(files, metadata, imgnames)
    srctext = proc.findAnchors(rawML, ncx_data, positionMap)
    srctext, usedmap = proc.insertHREFS()

    # write the proper mobi html
    fileinfo=[]
    fname = files.getInputFileBasename() + '.html'
    fileinfo.append([None,'', fname])
    outhtml = os.path.join(files.mobi7dir, fname)
    open(pathof(outhtml), 'wb').write(srctext)

    # extract guidetext from srctext
    guidetext =''
    pagemapxml = ''
    guidematch = re.search(r'''<guide>(.*)</guide>''',srctext,re.IGNORECASE+re.DOTALL)
    if guidematch:
        guidetext = guidematch.group(1)
        # sometimes old mobi guide from srctext horribly written so need to clean up
        guidetext = guidetext.replace("\r", "")
        guidetext = guidetext.replace('<REFERENCE', '<reference')
        guidetext = guidetext.replace(' HREF=', ' href=')
        guidetext = guidetext.replace(' TITLE=', ' title=')
        guidetext = guidetext.replace(' TYPE=', ' type=')
        # reference must be a self-closing tag
        # and any href must be replaced with filepos information
        ref_tag_pattern = re.compile(r'''(<reference [^>]*>)''', re.IGNORECASE)
        guidepieces = ref_tag_pattern.split(guidetext)
        for i in range(1,len(guidepieces), 2):
            reftag = guidepieces[i]
            # remove any href there now to replace with filepos
            reftag = re.sub(r'''href\s*=[^'"]*['"][^'"]*['"]''','', reftag)
            # make sure the reference tag ends properly
            if not reftag.endswith("/>"):
                reftag = reftag[0:-1] + "/>"
                guidepieces[i] = reftag
        guidetext = "".join(guidepieces)
        replacetext = r'''href="'''+fileinfo[0][2]+r'''#filepos\1"'''
        guidetext = re.sub(r'''filepos=['"]{0,1}0*(\d+)['"]{0,1}''', replacetext, guidetext)
        guidetext += '\n'
        if isinstance(guidetext, unicode):
            guidetext = guidetext.decode(mh.codec).encode("utf-8")
        else:
            guidetext = unicode(guidetext, mh.codec).encode("utf-8")

    if 'StartOffset' in metadata.keys():
        for value in metadata['StartOffset']:
            if int(value) == 0xffffffff:
                value = '0'
            starting_offset = value
        # get guide items from metadata
        metaguidetext = '<reference type="text" href="'+fileinfo[0][2]+'#filepos'+starting_offset+'" />\n'
        guidetext += metaguidetext

    # create an OPF
    opf = OPFProcessor(files, metadata, fileinfo, imgnames, ncx.isNCX, mh, usedmap, pagemapxml, guidetext)
    opf.writeOPF()


def processUnknownSections(mh, sect, files, K8Boundary):
    global DUMP
    global TERMINATION_INDICATOR1
    global TERMINATION_INDICATOR2
    global TERMINATION_INDICATOR3
    if DUMP:
        print "Unpacking any remaining unknown records"
    beg = mh.start
    end = sect.num_sections
    if beg < K8Boundary:
        # then we're processing the first part of a combination file
        end = K8Boundary
    for i in xrange(beg, end):
        if sect.sectiondescriptions[i] == "":
            data = sect.loadSection(i)
            type = data[0:4]
            if  type == TERMINATION_INDICATOR3:
                description = "Termination Marker 3 Nulls"
            elif type == TERMINATION_INDICATOR2:
                description = "Termination Marker 2 Nulls"
            elif type == TERMINATION_INDICATOR1:
                description = "Termination Marker 1 Null"
            elif type == "INDX":
                fname = "Unknown%05d_INDX.dat" % i
                description = "Unknown INDX section"
                if DUMP:
                    outname= os.path.join(files.outdir, fname)
                    open(pathof(outname), 'wb').write(data)
                    print "Extracting %s: %s from section %d" % (description, fname, i)
                    description = description + ", extracting as %s" % fname
            else:
                fname = "unknown%05d.dat" % i
                description = "Mysterious Section, first four bytes %s" % describe(data[0:4])
                if DUMP:
                    outname= os.path.join(files.outdir, fname)
                    open(pathof(outname), 'wb').write(data)
                    print "Extracting %s: %s from section %d" % (description, fname, i)
                    description = description + ", extracting as %s" % fname
            sect.setsectiondescription(i, description)


def process_all_mobi_headers(files, apnxfile, sect, mhlst, K8Boundary, k8only=False, epubver='2', use_hd=False):
    global DUMP
    global WRITE_RAW_DATA
    imgnames = []
    image_ptr = -1
    k8resc = None
    obfuscate_data = []
    for mh in mhlst:
        pagemapproc = None
        if mh.isK8():
            sect.setsectiondescription(mh.start,"KF8 Header")
            mhname = os.path.join(files.outdir,"header_K8.dat")
            print "Processing K8 section of book..."
        elif mh.isPrintReplica():
            sect.setsectiondescription(mh.start,"Print Replica Header")
            mhname = os.path.join(files.outdir,"header_PR.dat")
            print "Processing PrintReplica section of book..."
        else:
            if mh.version == 0:
                sect.setsectiondescription(mh.start, "PalmDoc Header".format(mh.version))
            else:
                sect.setsectiondescription(mh.start,"Mobipocket {0:d} Header".format(mh.version))
            mhname = os.path.join(files.outdir,"header.dat")
            print "Processing Mobipocket {0:d} section of book...".format(mh.version)

        if DUMP:
            # write out raw mobi header data
            open(pathof(mhname), 'wb').write(mh.header)

        # process each mobi header
        metadata = mh.getMetaData()
        mh.describeHeader(DUMP)
        if mh.isEncrypted():
            raise unpackException('Book is encrypted')

        pagemapproc = None

        # first handle all of the different resource sections:  images, resources, fonts, and etc
        # build up a list of image names to use to postprocess the ebook

        print "Unpacking images, resources, fonts, etc"
        beg = mh.firstresource
        end = sect.num_sections
        if beg < K8Boundary:
            # processing first part of a combination file
            end = K8Boundary

        cover_offset = int(metadata.get('CoverOffset', ['-1'])[0])
        if not CREATE_COVER_PAGE:
            cover_offset = None

        for i in xrange(beg, end):
            data = sect.loadSection(i)
            type = data[0:4]

            # handle the basics first
            if type in ["FLIS", "FCIS", "FDST", "DATP"]:
                if DUMP:
                    fname = type + "%05d" % i
                    if mh.isK8():
                        fname += "_K8"
                    fname += '.dat'
                    outname= os.path.join(files.outdir, fname)
                    open(pathof(outname), 'wb').write(data)
                    print "Dumping section {0:d} type {1:s} to file {2:s} ".format(i,type,outname)
                sect.setsectiondescription(i,"Type {0:s}".format(type))
                imgnames.append(None)
            elif type == "SRCS":
                imgnames = processSRCS(i, files, imgnames, sect, data)
            elif type == "PAGE":
                imgnames, pagemapproc = processPAGE(i, files, imgnames, sect, data, mh, pagemapproc)
            elif type == "CMET":
                imgnames = processCMET(i, files, imgnames, sect, data)
            elif type == "FONT":
                imgnames, obfuscate_data = processFONT(i, files, imgnames, sect, data, obfuscate_data)
            elif type == "CRES":
                imgnames, image_ptr = processCRES(i, files, imgnames, sect, data, beg, image_ptr, use_hd)
            elif type == "CONT":
                imgnames = processCONT(i, files, imgnames, sect, data)
            elif type == "kind":
                imgnames = processkind(i, files, imgnames, sect, data)
            elif type == chr(0xa0) + chr(0xa0) + chr(0xa0) + chr(0xa0):
                sect.setsectiondescription(i,"Empty_HD_Image/Resource_Placeholder")
                imgnames.append(None)
                image_ptr += 1
            elif type == "RESC":
                imgnames, k8resc = processRESC(i, files, imgnames, sect, data, k8resc)
            elif data == EOF_RECORD:
                sect.setsectiondescription(i,"End Of File")
                imgnames.append(None)
            elif data[0:8] == "BOUNDARY":
                sect.setsectiondescription(i,"BOUNDARY Marker")
                imgnames.append(None)
            else:
                # if reached here should be an image ow treat as unknown
                imgnames, image_ptr  = processImage(i, files, imgnames, sect, data, beg, image_ptr, cover_offset)
        # done unpacking resources

        # Print Replica
        if mh.isPrintReplica() and not k8only:
            processPrintReplica(metadata, files, imgnames, mh)
            continue

        # KF8 (Mobi 8)
        if mh.isK8():
            processMobi8(mh, metadata, sect, files, imgnames, pagemapproc, k8resc, obfuscate_data, apnxfile, epubver)

        # Old Mobi (Mobi 7)
        elif not k8only:
            processMobi7(mh, metadata, sect, files, imgnames)

        # process any remaining unknown sections of the palm file
        processUnknownSections(mh, sect, files, K8Boundary)

    return


def unpackBook(infile, outdir, apnxfile=None, epubver='2', use_hd=False, dodump=False, dowriteraw=False, dosplitcombos=False):
    global DUMP
    global WRITE_RAW_DATA
    global SPLIT_COMBO_MOBIS
    if DUMP or dodump:
        DUMP = True
    if WRITE_RAW_DATA or dowriteraw:
        WRITE_RAW_DATA = True
    if SPLIT_COMBO_MOBIS or dosplitcombos:
        SPLIT_COMBO_MOBIS = True

    infile = utf8_str(infile)
    outdir = utf8_str(outdir)
    if apnxfile is not None:
        apnxfile = utf8_str(apnxfile)

    files = fileNames(infile, outdir)

    # process the PalmDoc database header and verify it is a mobi
    sect = Sectionizer(infile)
    if sect.ident != 'BOOKMOBI' and sect.ident != 'TEXtREAd':
        raise unpackException('Invalid file format')
    if DUMP:
        sect.dumppalmheader()
    else:
        print "Palm DB type: %s, %d sections." % (sect.ident,sect.num_sections)

    # scan sections to see if this is a compound mobi file (K8 format)
    # and build a list of all mobi headers to process.
    mhlst = []
    mh = MobiHeader(sect,0)
    # if this is a mobi8-only file hasK8 here will be true
    mhlst.append(mh)
    K8Boundary = -1

    if mh.isK8():
        print "Unpacking a KF8 book..."
        hasK8 = True
    else:
        # This is either a Mobipocket 7 or earlier, or a combi M7/KF8
        # Find out which
        hasK8 = False
        for i in xrange(len(sect.sectionoffsets)-1):
            before, after = sect.sectionoffsets[i:i+2]
            if (after - before) == 8:
                data = sect.loadSection(i)
                if data == K8_BOUNDARY:
                    sect.setsectiondescription(i,"Mobi/KF8 Boundary Section")
                    mh = MobiHeader(sect,i+1)
                    hasK8 = True
                    mhlst.append(mh)
                    K8Boundary = i
                    break
        if hasK8:
            print "Unpacking a Combination M{0:d}/KF8 book...".format(mh.version)
            if SPLIT_COMBO_MOBIS:
                # if this is a combination mobi7-mobi8 file split them up
                mobisplit = mobi_split(infile)
                if mobisplit.combo:
                    outmobi7 = os.path.join(files.outdir, 'mobi7-'+files.getInputFileBasename() + '.mobi')
                    outmobi8 = os.path.join(files.outdir, 'mobi8-'+files.getInputFileBasename() + '.azw3')
                    open(pathof(outmobi7), 'wb').write(mobisplit.getResult7())
                    open(pathof(outmobi8), 'wb').write(mobisplit.getResult8())
        else:
            print "Unpacking a Mobipocket {0:d} book...".format(mh.version)

    if hasK8:
        files.makeK8Struct()

    process_all_mobi_headers(files, apnxfile, sect, mhlst, K8Boundary, False, epubver, use_hd)

    if DUMP:
        sect.dumpsectionsinfo()
    return


def usage(progname):
    print ""
    print "Description:"
    print "  Unpacks an unencrypted Kindle/MobiPocket ebook to html and images"
    print "  or an unencrypted Kindle/Print Replica ebook to PDF and images"
    print "  into the specified output folder."
    print "Usage:"
    print "  %s -r -s -p apnxfile -d -h --epub_version= infile [outdir]" % progname
    print "Options:"
    print "    -h                 print this help message"
    print "    -i                 use HD Images, if present, to overwrite reduced resolution images"
    print "    -s                 split combination mobis into mobi7 and mobi8 ebooks"
    print "    -p APNXFILE        path to an .apnx file associated with the azw3 input (optional)"
    print "    --epub_version=    specify epub version to unpack to: 2, 3, A (for automatic) or "
    print "                         F (force to fit to epub2 definitions), default is 2"
    print "    -d                 dump headers and other info to output and extra files"
    print "    -r                 write raw data to the output folder"


def main():
    global DUMP
    global WRITE_RAW_DATA
    global SPLIT_COMBO_MOBIS
    print "KindleUnpack v0.75"
    print "   Based on initial mobipocket version Copyright © 2009 Charles M. Hannum <root@ihack.net>"
    print "   Extensive Extensions and Improvements Copyright © 2009-2014 "
    print "       by:  P. Durrant, K. Hendricks, S. Siebert, fandrieu, DiapDealer, nickredding, tkeo."
    print "   This program is free software: you can redistribute it and/or modify"
    print "   it under the terms of the GNU General Public License as published by"
    print "   the Free Software Foundation, version 3."

    argv=utf8_argv()
    progname = os.path.basename(argv[0])
    try:
        opts, args = getopt.getopt(argv[1:], "dhirsp:", ['epub_version='])
    except getopt.GetoptError, err:
        print str(err)
        usage(progname)
        sys.exit(2)

    if len(args)<1:
        usage(progname)
        sys.exit(2)

    apnxfile = None
    epubver = '2'
    use_hd = False

    for o, a in opts:
        if o == "-h":
            usage(progname)
            sys.exit(0)
        if o == "-i":
            use_hd = True
        if o == "-d":
            DUMP = True
        if o == "-r":
            WRITE_RAW_DATA = True
        if o == "-s":
            SPLIT_COMBO_MOBIS = True
        if o == "-p":
            apnxfile = a
        if o == "--epub_version":
            epubver = a

    if len(args) > 1:
        infile, outdir = args
    else:
        infile = args[0]
        outdir = os.path.splitext(infile)[0]


    infileext = os.path.splitext(infile)[1].upper()
    if infileext not in ['.MOBI', '.PRC', '.AZW', '.AZW3', '.AZW4']:
        print "Error: first parameter must be a Kindle/Mobipocket ebook or a Kindle/Print Replica ebook."
        return 1

    try:
        print 'Unpacking Book...'
        unpackBook(infile, outdir, apnxfile, epubver, use_hd)
        print 'Completed'

    except ValueError, e:
        print "Error: %s" % e
        return 1

    return 0


if __name__ == '__main__':
    sys.exit(main())
