#!/usr/bin/python
#imports
import re
import os, sys, shutil
import tarfile
import hashlib
import tempfile
try:
    import markdown
except:
    print "ScrapeRepo: " + "markdown package is required but not found."
    print "ScrapeRepo: " + "install with `pip install markdown`"
    exit(0)
    
def displayHelp():
    print """
Usage: ScrapeRepo RepoDir DocDir RepoURL
  Create site files from the src repo
    where RepoDir is the directory containing example source core
          DocDir is the directory to receive the markdown pages
          RepoURL is the githib repo URL
"""
    exit(0)

def ConvertAndCopyFile(fromFile, toFile):
    mdFile = open(fromFile, 'r');
    htmlFile = open(toFile, 'w')
    md = mdFile.read()
    mdFile.close()
    html = html = markdown.markdown(md, extensions=['markdown.extensions.tables','markdown.extensions.toc(title=Contents)'])
    htmlFile.write(html)
    htmlFile.close()
    
def CopyFile(fromFile, toFile):
    mdFile = open(fromFile, 'r');
    outFile = open(toFile, 'w')
    md = mdFile.read()
    mdFile.close()
    outFile.write(md)
    outFile.close()
    
def GetVTKCMake(S):
    reg = re.compile(r"\{\{(VTKCMakeLists)\|(.*?)\}\}.*?", re.DOTALL)
    return [reg.findall(S)]

def GetVTKQtCMake(S):
    reg = re.compile(r"\{\{(VTKQtCMakeLists)\|(.*?)\}\}.*?", re.DOTALL)
    return [reg.findall(S)]

def GetVTKCMakelists(S):
    reg = re.compile(r"```cmake(.*?)```", re.DOTALL)
    return [''.join(reg.findall(S))]

def WriteCxxCode(toFile, codeName, code):
    toFile.write("**" + codeName + "**" + "\n")
    toFile.write("```c++" + "\n")
    toFile.write(code);
    toFile.write("```" + "\n")

# Fill in the template parameters in a CMakeLists template file
# The output is a CMakeLists.txt file with Name substituted for {{{1}}}
def FillCMakeLists(S, Name, ExtraNames):
    r1 = re.sub(r'XXX', Name, S)
    reg =  re.sub(r'YYY', ExtraNames, r1)
    return reg

def FillQtCMakeLists(S, Name):
    reg = re.sub(r'\{\{{2\}\}\}', Name, S)
    return reg

if len(sys.argv) < 4:
    displayHelp()
    exit(0)

# Get Repo and Wiki
RepoDir = sys.argv[1]
DocDir = sys.argv[2]
RepoURL = sys.argv[3]

# Make sure the wiki docs folder exists
if not os.path.exists("docs"):
    os.makedirs("docs")

BaselineName = RepoDir + "/Testing/Baseline"

# Read the Wiki Templates
VTKTemplateFile = open(RepoDir + "/Admin/VTKCMakeLists", 'r')
VTKTemplate = VTKTemplateFile.read();
VTKTemplateFile.close()

VTKQtTemplateFile = open(RepoDir + "/Admin/VTKQtCMakeLists", 'r')
VTKQtTemplate = VTKQtTemplateFile.read();
VTKQtTemplateFile.close()

# Make the reference mtime to be the most recent of the two CMakeLists templates
refStat1 = os.stat(RepoDir + "/Admin/VTKQtCMakeLists")
refStat2 = os.stat(RepoDir + "/Admin/VTKCMakeLists")
refMtime = refStat1.st_mtime
if refStat2.st_mtime > refStat1.st_mtime:
    refMtime = refStat2.st_mtime

# Create a dict to hold code name and page name
codeToPage = dict()

# Create a dict to hold CMakeLists.txt file
exampleToCMake = dict()

# Create a dict to hold the file names for each example
exampleToFileNames = dict()

# Convert repo .md files
CopyFile(RepoDir + "/index.md", DocDir + "/index.md")
CopyFile(RepoDir + "/Cxx.md", DocDir + "/Cxx.md")
CopyFile(RepoDir + "/Tcl.md", DocDir + "/Tcl.md")
CopyFile(RepoDir + "/Python.md", DocDir + "/Python.md")
CopyFile(RepoDir + "/Java.md", DocDir + "/Java.md")
CopyFile(RepoDir + "/CSharp.md", DocDir + "/CSharp.md")
if not os.path.exists(DocDir + "/Instructions"):
    os.makedirs(DocDir + "/Instructions")
CopyFile(RepoDir + "/Instructions/ForUsers.md", DocDir + "/Instructions/ForUsers.md")
CopyFile(RepoDir + "/Instructions/ForDevelopers.md", DocDir + "/Instructions/ForDevelopers.md")
CopyFile(RepoDir + "/Instructions/ForAdministrators.md", DocDir + "/Instructions/ForAdministrators.md")
CopyFile(RepoDir + "/Instructions/Guidelines.md", DocDir + "/Instructions/Guidelines.md")

# Get list of all  examples
Languages = dict()
Languages["Cxx"] = ".cxx"
Languages["Python"] = ".py"
Languages["Java"] = ".java"
Languages["CSharp"] = ".cs"
Languages["Tcl"] = ".tcl"

# Scan all Cxx directories and look for extras
allExtras = set()
for root, dirs, files in os.walk(RepoDir):
    to_find = RepoDir + "/" + "Cxx"
    start = root.find(to_find)
    if start < 0:
        continue
    for f in files:
        fileName = os.path.splitext(f)
        if fileName[1] == ".extras":
            print "Extra: " + root + "/" + f
            ExtrasPath = root + "/" + f
            ExtrasFile = open(ExtrasPath, 'r')
            for line in ExtrasFile:
                    line = line.strip()
                    allExtras.add(line)
            ExtrasFile.close()

for lang, langExt in Languages.items():
    for root, dirs, files in os.walk(RepoDir):
        to_find = RepoDir + "/" + lang
        start = root.find(to_find)
        if start < 0:
            continue
        print "root: " + root
        # Get the part of the file name that comes after RepoDir
        # e.g. if the file name is VTKExamples/Cxx/GeometricObjects/Line, 
        # Path will be Cxx/GeometriObjects/Line
        KitName = root[start+1+len(to_find):]

        if KitName.find("Boneyard") >= 0:
            continue
        if KitName.find("Broken") >= 0:
            continue
        for f in files:
            # skip files that are listed as extras
            if f in allExtras:
                print "Skipping: " + f
                continue
            if f == "CMakeLists.txt":
                continue
            ExampleName = os.path.splitext(f)[0]
            ExampleExt = os.path.splitext(f)[1]
            if ExampleExt != langExt:
                continue
            print "ExampleName: " + ExampleName + ExampleExt
            BaselinePath = RepoDir + "/Testing/Baseline/" + lang + "/" + KitName + "/Test" + ExampleName + ".png"
            PathName = DocDir + "/" + lang + "/" + KitName
            if not os.path.exists(PathName):
                if PathName != "":
                    os.makedirs(PathName)
            OutputFile = DocDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".md"
            MdFile = open(OutputFile, 'w')
            MdFile.write("[VTKExamples](/)/[" + lang + "](/" + lang + ")/" + KitName + "/" + ExampleName + "\n\n")
                    
            if os.path.isfile(BaselinePath) :
                print "BaselinePath: " + BaselinePath
                ImgUrl =  RepoURL + "/blob/master/src/Testing/Baseline/" + lang + "/" + KitName + "/Test" + ExampleName + ".png?raw=true"
                print "ImgUrl: " + ImgUrl
                MdFile.write('<img style="float:middle" src="' + ImgUrl + '" width="256" />' + '\n')
                MdFile.write("\n")
            DescriptionPath = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".md"
            if os.path.isfile(DescriptionPath) :
                DescriptionFile = open(RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".md", 'r')
                description = DescriptionFile.read()
                DescriptionFile.close()
                MdFile.write(description)
                MdFile.write("\n")
            MdFile.write("###Code\n")
            MdFile.write("**" + ExampleName + langExt + "**" + "\n")
            if langExt == ".cxx":
                MdFile.write("```c++" + "\n")
            elif langExt == ".cs":
                MdFile.write("```csharp" + "\n")
            elif langExt == ".py":
                MdFile.write("```python" + "\n")
            elif langExt == ".tcl":
                MdFile.write("```tcl" + "\n")
            elif langExt == ".java":
                MdFile.write("```java" + "\n")
            SrcFileName = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + langExt
            SrcFile = open(SrcFileName, 'r')
            src = SrcFile.read()
            SrcFile.close()
            MdFile.write(src)
            MdFile.write("```" + "\n")

            # Store the full file names for the example
            if ExampleName not in exampleToFileNames:
                exampleToFileNames[ExampleName] = set()
            SrcFile = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ExampleExt
            exampleToFileNames[ExampleName].add(SrcFile)

            ExtrasPath = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".extras"
            ExtraNames = ""
            if os.path.isfile(ExtrasPath) :
                print "Extras.. " + ExtrasPath + "\n"
                ExtrasFile = open(ExtrasPath, 'r')
                for line in ExtrasFile:
                    line = line.strip()
                    print line
                    ExtraPath = RepoDir + "/" + lang + "/" + KitName + "/" + line
                    SrcFile = RepoDir + "/" + lang + "/" + KitName + "/" + line

                    exampleToFileNames[ExampleName].add(SrcFile)
                    print "\t" + "ExtraPath: " + ExtraPath
                    ExtraFile = open(ExtraPath, 'r')
                    extraCode = ExtraFile.read()
                    ExtraFile.close()
                    WriteCxxCode(MdFile, line, extraCode)
                    extent = os.path.splitext(line)
                    if extent[1] == ".cxx":
                        ExtraNames += " " + line
                ExtrasFile.close()
            CustomCMakePath = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".cmake"
            if os.path.isfile(CustomCMakePath) :
                CustomCMakeFile = open(CustomCMakePath, 'r')
                cmake = CustomCMakeFile.read()
                CustomCMakeFile.close()
            else:
                CMakeFile = open(RepoDir + "/Admin/VTKCMakeLists", 'r')
                CMakeContents = CMakeFile.read()
                CMakeFile.close()
                cmake = FillCMakeLists(CMakeContents, ExampleName, ExtraNames)
            exampleToCMake[ExampleName] = GetVTKCMakelists(cmake)
            if lang == "Cxx": 
                MdFile.write(cmake)
            MdFile.close()
            codeToPage[ExampleName + langExt] = "/" + lang + "/" + KitName + "/" + ExampleName

# Generate an html page that links each example code file to its Wiki Example
# page
indexFile = open(DocDir + "/ExampleCodeToWikiPage.html", 'w')
indexFile.write("Navigate to the page that contains the source code of an example<br>");
indexFile.write("\n");
sortedByCode = sorted(codeToPage.items())
for item in sortedByCode:
    indexFile.write("<A HREF=" + RepoURL + "/wikis" + re.sub(" ", "_", item[1]) + ">" + item[0] + "</A>")
    indexFile.write("<A HREF=" + RepoURL + "/blob/master" + re.sub(" ", "_", item[1]) + ".md" + ">" + "(md)" + "</A>")
    indexFile.write("<br>\n")
indexFile.close()

# Create tarballs for each example
tmpDir = tempfile.mkdtemp(prefix="VTKTarballs") + "/"

# Create the Tarballs directory in the source tree if not present
# If it does not exist, assume the tarball repo has not been cloned
# and we need to ignore tar files
if not os.path.exists("src/Tarballs"):
    os.makedirs("src/Tarballs")
    ignoreFile = open("src/Tarballs/.gitignore", 'w')
    ignoreFile.write("*,tar\n")
    ignoreFile.close()

# Create tarballs
# For each example page, create a directory and copy that example's files
# into the directory
# If the example has a CMakeLists.txt file, copy that.
# Also, create a subdir called build. This directory is handy when you want to
# configure with CMake and build the example.
for example in exampleToFileNames:
    if example not in exampleToCMake:
        continue
    # Make the directories for the example
    srcDir = tmpDir + example
    codeFileName = srcDir + "/" + example + ".cxx"
    if not os.path.exists(srcDir):
        os.makedirs(srcDir)
        os.makedirs(srcDir + "/build")

        # An example may have multiple source files
        for exampleFileName in exampleToFileNames[example]:
            # Get the file name
            tarFileName = srcDir + "/" + os.path.split(exampleFileName)[1]
            # Copy the code for the example
            shutil.copy(exampleFileName,tarFileName)
            os.utime(tarFileName,(refMtime,refMtime))

    # Some examples do not have a CMakeLists.txt file
    if example in exampleToCMake:
        cmakeFileName = srcDir + "/" + "CMakeLists.txt"
        cmakeFile = open(cmakeFileName, 'w')
        cmakeFile.write(exampleToCMake[example][0])    
        cmakeFile.close()
        os.utime(cmakeFileName,(refMtime,refMtime))

    # Set the mtimes for the directories containing the example
    # Since the mtime is stored in the tar header for each file and directory,
    # we need a consistent time so that a tar of an unchanged example will be equal
    # to the one in the repo
    os.utime(srcDir,(refMtime,refMtime))
    os.utime(srcDir + "/build",(refMtime,refMtime))

    # Now create the tar file for the example
    # The tarballs are stored in the source tree
    tar = tarfile.open("src/Tarballs/" + example + ".tar", "w")
    tar.add(srcDir,arcname=example)
    tar.close()

os.utime(tmpDir,(0,refMtime))
# Cleanup the temporary directories
shutil.rmtree(tmpDir)
