#!/usr/bin/python
#imports
import re
import os, sys, shutil
import tarfile
import hashlib
import tempfile
try:
    import tinyurl
except:
    print "ScrapeRepo: " + "tinyurl package is required but not found."
    print "ScrapeRepo: " + "install with `pip install tinyurl`"
    exit(0)

try:
    import markdown
except:
    print "ScrapeRepo: " + "markdown package is required but not found."
    print "ScrapeRepo: " + "install with `pip install markdown`"
    exit(0)
    
def displayHelp():
    print """
Usage: ScrapeRepo RepoDir DocDir RepoURL
  Create site files from the src repo
    where RepoDir is the directory containing example source core
          DocDir is the directory to receive the markdown pages
          RepoURL is the githib repo URL
"""
    exit(0)

# convert a file from markdown to html
def ConvertAndCopyFile(fromFile, toFile):
    mdFile = open(fromFile, 'r');
    htmlFile = open(toFile, 'w')
    md = mdFile.read()
    mdFile.close()
    html = markdown.markdown(md, extensions=['markdown.extensions.tables','markdown.extensions.toc(title=Contents)'])
    htmlFile.write(html)
    htmlFile.close()
    
# copy a file
def CopyFile(fromFile, toFile):
    mdFile = open(fromFile, 'r');
    outFile = open(toFile, 'w')
    md = mdFile.read()
    mdFile.close()
    outFile.write(md)
    outFile.close()
    
# load the tiny url cache into a dictionary
def LoadTinyUrlCache (CacheFile, CacheDict) :
    cf = open(CacheFile, 'r')
    for line in cf:
        words = line.split()
        CacheDict[words[0]] = words[1]
    cf.close()

# if the url is not in the cache, get the tinyurl
def GetTinyUrl(CacheDict, Url):
    global cacheHits
    global cacheMisses
    if Url in CacheDict.keys() :
        cacheHits = cacheHits + 1
        return CacheDict[Url]
    tinyOne = tinyurl.create_one(Url)
    CacheDict[Url] = tinyOne;
    cacheMisses = cacheMisses + 1
    print "cache miss: " + Url
    return CacheDict[Url]
    
def IsQtExample(S):
    reg = re.compile(r".*Qt.*", re.DOTALL)
    return reg.match(S)

def GetVTKQtCMake(S):
    reg = re.compile(r"\{\{(VTKQtCMakeLists)\|(.*?)\}\}.*?", re.DOTALL)
    return [reg.findall(S)]

def GetVTKCMakelists(S):
    reg = re.compile(r"```cmake(.*?)```", re.DOTALL)
    return [''.join(reg.findall(S))]

# add header and code hilighting to c++ code
def WriteCxxCode(toFile, codeName, code):
    toFile.write("**" + codeName + "**" + "\n")
    toFile.write("```c++" + "\n")
    toFile.write(code);
    toFile.write("```" + "\n")

# Return hl_lines to highlight lines that have vtk classes mentiones
def FindLinesWithVTK(srcFileName):
    srcFile = open(srcFileName, 'r')
    lines = srcFile.readlines()
    lineNumber = 1
    reg = re.compile(r'(vtk[0-9a-zA-Z]*)')
    hl_lines = 'hl_lines="'
    for line in lines:
        if reg.search(line):
            hl_lines += str(lineNumber) + " "
        lineNumber += 1
    hl_lines += '"\n'
    srcFile.close()
    return hl_lines

def AddDoxygen(S):
    global doxyCount
    reg = re.compile(r'[^\./\[s](vtk[0-9a-zA-Z]*)')
    if reg.findall(S):
        doxyCount = doxyCount + 1
        return re.sub(r'[^\./\[s](vtk[0-9a-zA-Z]*)', r' [\1](' + r'http://www.vtk.org/doc/nightly/html/class' + r'\1.html#details)', S)
    return S

# add doxygen links to a file
def AddDoxygens(repoDir, repoURL, fromFile, toFile):
    mdFile = open(fromFile, 'r');
    outFile = open(toFile, 'w')
    for line in mdFile:
        withDoxy = AddDoxygen(line)
        outFile.write(withDoxy.rstrip())        
        outFile.write("\n")
    mdFile.close()
    outFile.close()

# add thumbnails to example tables
def FindThumbnail(S):
        reg = re.compile(r'^\[[^\(]*\]\(([^)]*)')
        if reg.match(S):
                return [''.join(reg.findall(S))]
        return['']

# add thumbnails to a file
def AddThumbnails(repoDir, repoURL, fromFile, toFile):
    global thumbCount
    mdFile = open(fromFile, 'r');
    outFile = open(toFile, 'w')
    for line in mdFile:
        withDoxy = AddDoxygen(line)
        outFile.write(withDoxy.rstrip())        
        exampleLine = FindThumbnail(line.strip())[0]
        if  exampleLine != '':
            thumbCount = thumbCount + 1
            exampleName = os.path.split(exampleLine)[1]
            exampleDir = os.path.split(exampleLine)[0]
            baseline = RepoDir + "/Testing/Baseline" + exampleDir + "/Test" + exampleName + ".png"
            baselineURL = repoURL + "/blob/master/" + baseline
            tinyBaseline = GetTinyUrl(CacheDict, baselineURL)
            if os.path.exists(baseline):
                img = '<img class="lazy" style="float:center" data-src="' + tinyBaseline + '?raw=true" width="64" />'
                outFile.write(" | ")
                outFile.write('<a href="' + baselineURL + '?raw=true target="_blank">')
                outFile.write(img)
                outFile.write("\n</a>")
        outFile.write("\n")
    mdFile.close()
    outFile.close()

# Fill in the template parameters in a CMakeLists template file
# The output is a CMakeLists.txt file with Name substituted for {{{1}}}
def FillCMakeLists(S, Name, ExtraNames):
    r1 = re.sub(r'XXX', Name, S)
    reg =  re.sub(r'YYY', ExtraNames, r1)
    return reg

def FillQtCMakeLists(S, Name):
    reg = re.sub(r'XXX', Name, S)
    return reg

if len(sys.argv) < 4:
    displayHelp()
    exit(0)

# Get Repo and Wiki
RepoDir = sys.argv[1]
DocDir = sys.argv[2]
RepoURL = sys.argv[3]
cacheHits = 0;
cacheMisses = 0;

cxxCount = 0;
csCount = 0;
pyCount = 0;
javaCount = 0;

thumbCount = 0
doxyCount = 0
# Make sure the wiki docs folder exists
if not os.path.exists("docs"):
    os.makedirs("docs")

BaselineName = RepoDir + "/Testing/Baseline"

# Load the TinyUrl cache
CacheFile = RepoDir + "/Admin/TinyUrlCache"
CacheDict = dict()
LoadTinyUrlCache(CacheFile, CacheDict)

# Read the Wiki Templates
VTKTemplateFile = open(RepoDir + "/Admin/VTKCMakeLists", 'r')
VTKTemplate = VTKTemplateFile.read();
VTKTemplateFile.close()

VTKQtTemplateFile = open(RepoDir + "/Admin/VTKQtCMakeLists", 'r')
VTKQtTemplate = VTKQtTemplateFile.read();
VTKQtTemplateFile.close()

# Make the reference mtime to be the most recent of the two CMakeLists templates
refStat1 = os.stat(RepoDir + "/Admin/VTKQtCMakeLists")
refStat2 = os.stat(RepoDir + "/Admin/VTKCMakeLists")
refMtime = refStat1.st_mtime
if refStat2.st_mtime > refStat1.st_mtime:
    refMtime = refStat2.st_mtime

# Create a dict to hold code name and page name
codeToPage = dict()

# Create a dict to hold CMakeLists.txt file
exampleToCMake = dict()

# Create a dict to hold the file names for each example
exampleToFileNames = dict()

if not os.path.exists(DocDir + "/Cxx/Snippets"):
    os.makedirs(DocDir + "/Cxx/Snippets")
AddThumbnails(RepoDir, RepoURL, RepoDir + "/Cxx.md", DocDir + "/Cxx.md")
AddThumbnails(RepoDir, RepoURL, RepoDir + "/Python.md", DocDir + "/Python.md")
AddThumbnails(RepoDir, RepoURL, RepoDir + "/CSharp.md", DocDir + "/CSharp.md")
AddThumbnails(RepoDir, RepoURL, RepoDir + "/Java.md", DocDir + "/Java.md")
AddThumbnails(RepoDir, RepoURL, RepoDir + "/Cxx/Snippets.md", DocDir + "/Cxx/Snippets.md")
AddThumbnails(RepoDir, RepoURL, RepoDir + "/VTKBookFigures.md", DocDir + "/VTKBookFigures.md")

# Snippets
snippets = ['ReadPolyData', 'SaveSceneToFieldData', 'SaveSceneToFile', 'RestoreSceneFromFieldData', 'RestoreSceneFromFile', 'ChooseContrastingColor']
for snippet in snippets:
    CopyFile(RepoDir + '/Cxx/Snippets/' + snippet + '.md', DocDir + '/Cxx/Snippets/' + snippet + '.md')

# Convert repo .md files
CopyFile(RepoDir + "/index.md", DocDir + "/index.md")

if not os.path.exists(DocDir + "/Coverage"):
    os.makedirs(DocDir + "/Coverage")
CopyFile(RepoDir + "/Coverage/CxxVTKClassesNotUsed.md", DocDir + "/Coverage/CxxVTKClassesNotUsed.md")
CopyFile(RepoDir + "/Coverage/CxxVTKClassesUsed.md", DocDir + "/Coverage/CxxVTKClassesUsed.md")
CopyFile(RepoDir + "/Coverage/PythonVTKClassesNotUsed.md", DocDir + "/Coverage/PythonVTKClassesNotUsed.md")
CopyFile(RepoDir + "/Coverage/PythonVTKClassesUsed.md", DocDir + "/Coverage/PythonVTKClassesUsed.md")
CopyFile(RepoDir + "/Coverage/JavaVTKClassesNotUsed.md", DocDir + "/Coverage/JavaVTKClassesNotUsed.md")
CopyFile(RepoDir + "/Coverage/JavaVTKClassesUsed.md", DocDir + "/Coverage/JavaVTKClassesUsed.md")
CopyFile(RepoDir + "/Coverage/CSharpVTKClassesNotUsed.md", DocDir + "/Coverage/CSharpVTKClassesNotUsed.md")
CopyFile(RepoDir + "/Coverage/CSharpVTKClassesUsed.md", DocDir + "/Coverage/CSharpVTKClassesUsed.md")

if not os.path.exists(DocDir + "/Instructions"):
    os.makedirs(DocDir + "/Instructions")
CopyFile(RepoDir + "/Instructions/ForUsers.md", DocDir + "/Instructions/ForUsers.md")
CopyFile(RepoDir + "/Instructions/ForDevelopers.md", DocDir + "/Instructions/ForDevelopers.md")
CopyFile(RepoDir + "/Instructions/ForAdministrators.md", DocDir + "/Instructions/ForAdministrators.md")
CopyFile(RepoDir + "/Instructions/Guidelines.md", DocDir + "/Instructions/Guidelines.md")
CopyFile(RepoDir + "/Instructions/ConvertingFiguresToExamples.md", DocDir + "/Instructions/ConvertingFiguresToExamples.md")

# Get list of all  examples
Languages = dict()
Languages["Cxx"] = ".cxx"
Languages["Python"] = ".py"
Languages["Java"] = ".java"
Languages["CSharp"] = ".cs"

# Scan all Cxx directories and look for extras
allExtras = set()
for root, dirs, files in os.walk(RepoDir):
    to_find = RepoDir + "/" + "Cxx"
    start = root.find(to_find)
    if start < 0:
        continue
    for f in files:
        fileName = os.path.splitext(f)
        if fileName[1] == ".extras":
            ExtrasPath = root + "/" + f
            ExtrasFile = open(ExtrasPath, 'r')
            for line in ExtrasFile:
                    line = line.strip()
                    allExtras.add(line)
            ExtrasFile.close()

for lang, langExt in Languages.items():
    for root, dirs, files in os.walk(RepoDir):
        to_find = RepoDir + "/" + lang
        start = root.find(to_find)
        if start < 0:
            continue
        # Get the part of the file name that comes after RepoDir
        # e.g. if the file name is VTKExamples/Cxx/GeometricObjects/Line, 
        # Path will be Cxx/GeometriObjects/Line
        KitName = root[start+1+len(to_find):]

        if KitName.find("Boneyard") >= 0:
            continue
        if KitName.find("Broken") >= 0:
            continue
        for f in files:
            # skip files that are listed as extras
            if f in allExtras:
                continue
            if f == "CMakeLists.txt":
                continue
            ExampleName = os.path.splitext(f)[0]
            ExampleExt = os.path.splitext(f)[1]
            if ExampleExt != langExt:
                continue
            BaselinePath = RepoDir + "/Testing/Baseline/" + lang + "/" + KitName + "/Test" + ExampleName + ".png"
            PathName = DocDir + "/" + lang + "/" + KitName
            if not os.path.exists(PathName):
                if PathName != "":
                    os.makedirs(PathName)
            OutputFile = DocDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".md"
            MdFile = open(OutputFile, 'w')
            MdFile.write("[VTKExamples](/)/[" + lang + "](/" + lang + ")/" + KitName + "/" + ExampleName + "\n\n")
                    
            if os.path.isfile(BaselinePath) :
                ImgUrl =  RepoURL + "/blob/master/src/Testing/Baseline/" + lang + "/" + KitName + "/Test" + ExampleName + ".png?raw=true"
                # href to open image in new tab
                MdFile.write('<a href="' + ImgUrl + ' target="_blank">' + '\n')
                MdFile.write('<img style="border:2px solid beige;float:center" src="' + ImgUrl + '" width="256" />' + '\n')
                MdFile.write("</a>" + "\n")
                MdFile.write("<hr>\n")
                MdFile.write("\n")
            DescriptionPath = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".md"
            if os.path.isfile(DescriptionPath) :
                DescriptionFile = open(RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".md", 'r')
                description = DescriptionFile.read()
                DescriptionFile.close()
                description = AddDoxygen(description)
                MdFile.write(description)
                MdFile.write("\n")
            SrcFileName = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + langExt
            SrcFile = open(SrcFileName, 'r')
            src = SrcFile.read()
            SrcFile.close()
            hiliteLines = FindLinesWithVTK(SrcFileName)
            MdFile.write("###Code\n")
            MdFile.write("**" + ExampleName + langExt + "**" + "\n")
            if langExt == ".cxx":
                MdFile.write("``` c++ " + hiliteLines + "\n")
                cxxCount = cxxCount + 1
            elif langExt == ".cs":
                MdFile.write("```csharp" + hiliteLines + "\n")
                csCount = csCount + 1
            elif langExt == ".py":
                MdFile.write("```python" + hiliteLines + "\n")
                pyCount = pyCount + 1
            elif langExt == ".java":
                MdFile.write("```java" + hiliteLines + "\n")
                javaCount = javaCount + 1
            MdFile.write(src)
            MdFile.write("```" + "\n")
            
            # Store the full file names for the example
            if ExampleName not in exampleToFileNames:
                exampleToFileNames[ExampleName] = set()
            SrcFile = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ExampleExt
            exampleToFileNames[ExampleName].add(SrcFile)

            ExtrasPath = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".extras"
            ExtraNames = ""
            if os.path.isfile(ExtrasPath) :
                ExtrasFile = open(ExtrasPath, 'r')
                for line in ExtrasFile:
                    line = line.strip()
                    if line == '' :
                        continue
                    ExtraPath = RepoDir + "/" + lang + "/" + KitName + "/" + line
                    SrcFile = RepoDir + "/" + lang + "/" + KitName + "/" + line

                    exampleToFileNames[ExampleName].add(SrcFile)
                    ExtraFile = open(ExtraPath, 'r')
                    extraCode = ExtraFile.read()
                    ExtraFile.close()
                    WriteCxxCode(MdFile, line, extraCode)
                    extent = os.path.splitext(line)
                    if extent[1] == ".cxx":
                        ExtraNames += " " + line
                ExtrasFile.close()
            CustomCMakePath = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".cmake"
            if os.path.isfile(CustomCMakePath) :
                CustomCMakeFile = open(CustomCMakePath, 'r')
                cmake = CustomCMakeFile.read()
                CustomCMakeFile.close()
            else:
                if IsQtExample(src) :
                    CMakeFile = open(RepoDir + "/Admin/VTKQtCMakeLists", 'r')
                    CMakeContents = CMakeFile.read()
                    CMakeFile.close()
                    cmake = FillQtCMakeLists(CMakeContents, ExampleName)
                else :
                    CMakeFile = open(RepoDir + "/Admin/VTKCMakeLists", 'r')
                    CMakeContents = CMakeFile.read()
                    CMakeFile.close()
                    cmake = FillCMakeLists(CMakeContents, ExampleName, ExtraNames)
            if lang == "Cxx": 
                exampleToCMake[ExampleName] = GetVTKCMakelists(cmake)
                MdFile.write(cmake)
            MdFile.close()
            codeToPage[ExampleName + langExt] = "/" + lang + "/" + KitName + "/" + ExampleName

# Generate an html page that links each example code file to its Wiki Example
# page
indexFile = open(DocDir + "/ExampleCodeToWikiPage.html", 'w')
indexFile.write("Navigate to the page that contains the source code of an example<br>");
indexFile.write("\n");
sortedByCode = sorted(codeToPage.items())
for item in sortedByCode:
    indexFile.write("<A HREF=" + RepoURL + "/wikis" + re.sub(" ", "_", item[1]) + ">" + item[0] + "</A>")
    indexFile.write("<A HREF=" + RepoURL + "/blob/master" + re.sub(" ", "_", item[1]) + ".md" + ">" + "(md)" + "</A>")
    indexFile.write("<br>\n")
indexFile.close()

# Create tarballs for each example
tmpDir = tempfile.mkdtemp(prefix="VTKTarballs") + "/"

# Create the Tarballs directory in the source tree if not present
# If it does not exist, assume the tarball repo has not been cloned
# and we need to ignore tar files
if not os.path.exists("src/Tarballs"):
    os.makedirs("src/Tarballs")
    ignoreFile = open("src/Tarballs/.gitignore", 'w')
    ignoreFile.write("*,tar\n")
    ignoreFile.close()

# Create tarballs
# For each example page, create a directory and copy that example's files
# into the directory
# If the example has a CMakeLists.txt file, copy that.
# Also, create a subdir called build. This directory is handy when you want to
# configure with CMake and build the example.
for example in exampleToFileNames:
    if example not in exampleToCMake:
        continue
    # Make the directories for the example
    srcDir = tmpDir + example
    codeFileName = srcDir + "/" + example + ".cxx"
    if not os.path.exists(srcDir):
        os.makedirs(srcDir)

        # An example may have multiple source files
        for exampleFileName in exampleToFileNames[example]:
            # Get the file name
            tarFileName = srcDir + "/" + os.path.split(exampleFileName)[1]
            # Copy the code for the example
            shutil.copy(exampleFileName,tarFileName)
            os.utime(tarFileName,(refMtime,refMtime))

    # Some examples do not have a CMakeLists.txt file
    if example in exampleToCMake:
        os.makedirs(srcDir + "/build")
        cmakeFileName = srcDir + "/" + "CMakeLists.txt"
        cmakeFile = open(cmakeFileName, 'w')
        cmakeFile.write(exampleToCMake[example][0])    
        cmakeFile.close()
        os.utime(cmakeFileName,(refMtime,refMtime))

    # Set the mtimes for the directories containing the example
    # Since the mtime is stored in the tar header for each file and directory,
    # we need a consistent time so that a tar of an unchanged example will be equal
    # to the one in the repo
    os.utime(srcDir,(refMtime,refMtime))
    os.utime(srcDir + "/build",(refMtime,refMtime))

    # Now create the tar file for the example
    # The tarballs are stored in the source tree
    tar = tarfile.open("src/Tarballs/" + example + ".tar", "w")
    tar.add(srcDir,arcname=example)
    tar.close()

os.utime(tmpDir,(0,refMtime))
# Cleanup the temporary directories
shutil.rmtree(tmpDir)


# Update the cache file if necessary
if cacheMisses > 0 :
    cf = open(CacheFile, 'w')
    for key in CacheDict:
        cf.write(key + " " + CacheDict[key] + "\n")
    cf.close()

# Report stats
print "ScrapeRepo Summary"
print "    C++ examples: " + str(cxxCount)
print "    CSharp examples: " + str(csCount)
print "    Python examples: " + str(pyCount)
print "    Java examples: " + str(javaCount)
print "    Total examples: " + str(cxxCount + csCount + pyCount + javaCount)
print "    Doxygen added: " + str(doxyCount)
print "    Thumbnails added: " + str(thumbCount)
print "    TinyUrl Cache hits: " + str(cacheHits)
print "    TinyUrl Cache misses: " + str(cacheMisses)
