#!/usr/bin/python
# imports
from __future__ import print_function
import re
import os
import sys
import shutil
import tarfile
import hashlib
import tempfile
try:
    import tinyurl
except:
    print("ScrapeRepo: " + "tinyurl package is required but not found.")
    print("ScrapeRepo: " + "install with `pip install tinyurl`")
    exit(0)

try:
    import markdown
except:
    print("ScrapeRepo: " + "markdown package is required but not found.")
    print("ScrapeRepo: " + "install with `pip install markdown`")
    exit(0)


def displayHelp():
    print("""
Usage: ScrapeRepo RepoDir DocDir RepoURL
  Create site files from the src repo
    where RepoDir is the directory containing example source core
          DocDir is the directory to receive the markdown pages
          RepoURL is the githib repo URL
""")
    exit(0)

# copy a file

def CopyFile(fromFile, toFile):
    mdFile = open(fromFile, 'r')
    outFile = open(toFile, 'w')
    md = mdFile.read()
    mdFile.close()
    outFile.write(md)
    outFile.close()

# create a list of html id's
def CreateHtmlIds(fromFile):
    global htmlIdSet
    with open(fromFile, 'r') as chapterLine:
        for line in chapterLine:
            idFound = re.findall(r'<figure[ ]*id="([^\"]*)">', line)
            if len(idFound) > 0:
                htmlIdSet.add(idFound[0])

# copy a file and add figure and doxygen links
# only add links to doxygen if we are outside code blocks
# delimited by ```
# only add links to a figure if the figure exists
def CopyChapterAndAddLinks(fromFile, toFile):
    global htmlIdSet
    inCode = False
    outFile = open(toFile, 'w')
    with open(fromFile, 'r') as chapterLine:
        for line in chapterLine:
            if line.count(r'```') % 2 != 0 :
                inCode = not inCode
            if not inCode :
                line = AddDoxygen(line)
                figureFound = sorted(re.findall(r'\*\*(Figure[^\*]*)\*\*', line), reverse=True)
                if len(figureFound) > 0:
                    for figure in figureFound:
                        if figure in htmlIdSet :
                            line = line.replace(figure, r'<a href="#' + figure.replace("Figure","FIGURE") + '">**'+ figure.replace("Figure","FIGURE") + r'**</a>')
            line = line.replace("FIGURE", "Figure")
            outFile.write(line)
    outFile.close()

# load the tiny url cache into a dictionary

def LoadTinyUrlCache(CacheFile, CacheDict):
    cf = open(CacheFile, 'r')
    for line in cf:
        words = line.split()
        CacheDict[words[0]] = words[1]
    cf.close()

# if the url is not in the cache, get the tinyurl

def GetTinyUrl(CacheDict, Url):
    global cacheHits
    global cacheMisses
    if Url in CacheDict.keys():
        cacheHits = cacheHits + 1
        return CacheDict[Url]
    tinyOne = tinyurl.create_one(Url)
    CacheDict[Url] = tinyOne
    cacheMisses = cacheMisses + 1
    print("cache miss: " + Url)
    return CacheDict[Url]


# Is the example a Qt example

def IsQtExample(S):
    reg = re.compile(r".*Qt.*", re.DOTALL)
    return reg.match(S)

# Get the Qt CMake file

def GetVTKQtCMake(S):
    reg = re.compile(r"\{\{(VTKQtCMakeLists)\|(.*?)\}\}.*?", re.DOTALL)
    return [reg.findall(S)]

# Get the VTK CMake file
def GetVTKCMakelists(S):
    reg = re.compile(r"```cmake(.*?)```", re.DOTALL)
    return [''.join(reg.findall(S))]

# add header and code hilighting to c++ code

def WriteCxxCode(toFile, codeName, code):
    toFile.write("**" + codeName + "**" + "\n")
    toFile.write("```c++" + "\n")
    toFile.write(code)
    toFile.write("```" + "\n")

# Return hl_lines to highlight lines that have vtk classes mentiones

def FindLinesWithVTK(srcFileName):
    srcFile = open(srcFileName, 'r')
    lines = srcFile.readlines()
    lineNumber = 1
    reg = re.compile(r'(vtk[0-9a-zA-Z]*)')
    hl_lines = 'hl_lines="'
    for line in lines:
        if reg.search(line):
            hl_lines += str(lineNumber) + " "
        lineNumber += 1
    hl_lines += '"\n'
    srcFile.close()
    return hl_lines


# Add a link to an example in anpther language
def AddLanguage(S, link):
    reg = re.compile(r'(^\[[^\)]*\))')
    if reg.findall(S):
        return re.sub(r'(^\[[^\)]*\))', r'\1' + link, S)
    return S

# Given a lang example return a link to a Python example if it exists
def FindPythonGivenLang(repoDir, example, lang):
    if lang == "Python" : return ""
    pythonLink = re.sub(r'/' + lang + r'/',r'/Python/', example)
    pythonPath = repoDir + pythonLink + ".py"
    if os.path.exists(pythonPath) :
       return "([python](" + pythonLink + "))"
    return ""

# Given a lang example return a link to another example if it exists
def FindOtherGivenLang(repoDir, example, exampleLang, otherLang, otherExt):
    if otherLang == exampleLang : return ""
    otherLink = re.sub(r'/' + exampleLang + r'/',r'/' + otherLang + '/', example)
    otherPath = repoDir + otherLink + otherExt
    if os.path.exists(otherPath) :
       return "([" + otherLang + "](" + otherLink + "))"
    return ""

# Given a Cxx example return a link to a Python example if it exists
def FindPythonGivenCxx(repoDir, cxxExample):
    pythonLink = re.sub(r'/Cxx/',r'/Python/', cxxExample)
    pythonPath = repoDir + pythonLink + ".py"
    if os.path.exists(pythonPath) :
       return "([python](" + pythonLink + "))"
    return ""

# Given a Python example return a link to a Cxx example if it exists
def FindCxxGivenPython(repoDir, pythonExample):
    cxxLink = re.sub(r'/Python/',r'/Cxx/', pythonExample)
    cxxPath = repoDir + cxxLink + ".cxx"
    if os.path.exists(cxxPath) :
       return "([cxx](" + cxxLink + "))"
    return ""

# If vtkXXXX is in the string, add a link to the doxygen file

def AddDoxygen(S):
    global doxyCount
    reg = re.compile(r'[^\./\[s\-](vtk[^ &][0-9a-zA-Z]*)')
    if reg.findall(S):
        doxyCount = doxyCount + 1
        return re.sub(r'[^\./\[-](vtk[0-9a-zA-Z]*)', r' [\1](' + r'http://www.vtk.org/doc/nightly/html/class' + r'\1.html#details)', S)
    return S

# add doxygen links to a file

def AddDoxygens(repoDir, repoURL, fromFile, toFile):
    mdFile = open(fromFile, 'r')
    outFile = open(toFile, 'w')
    for line in mdFile:
        withDoxy = AddDoxygen(line)
        outFile.write(withDoxy.rstrip())
        outFile.write("\n")
    mdFile.close()
    outFile.close()

# add thumbnails to example tables

def FindThumbnail(S):
    reg = re.compile(r'^\[[^\(]*\]\(([^)]*)')
    if reg.match(S):
        return [''.join(reg.findall(S))]
    return['']

# add thumbnails to a file

def AddThumbnailsAndLanguageLinks(repoDir, repoURL, fromFile, toFile):
    global thumbCount
    mdFile = open(fromFile, 'r')
    outFile = open(toFile, 'w')
    for line in mdFile:
        exampleLine = FindThumbnail(line.strip())[0]
        startCxx = exampleLine.find("Cxx")
        if startCxx >= 0 :
            pythonLink = FindPythonGivenCxx("src", exampleLine)
            if pythonLink != '' :
                line = AddLanguage(line, pythonLink)

        startPython = exampleLine.find("Python")
        if startPython >= 0 :
            cxxLink = FindCxxGivenPython("src", exampleLine)
            if cxxLink != '' :
                line = AddLanguage(line, cxxLink)

        withDoxy = AddDoxygen(line)
        outFile.write(withDoxy.rstrip())
        if exampleLine != '':
            thumbCount = thumbCount + 1
            exampleName = os.path.split(exampleLine)[1]
            exampleDir = os.path.split(exampleLine)[0]
            baseline = RepoDir + "/Testing/Baseline" + exampleDir + "/Test" + exampleName + ".png"
            baselineURL = repoURL + "/blob/master/" + baseline
            tinyBaseline = GetTinyUrl(CacheDict, baselineURL)
            if os.path.exists(baseline):
                img = '<img class="lazy" style="float:center" data-src="' + tinyBaseline + '?raw=true" width="64" />'
                outFile.write(" | ")
                outFile.write('<a href="' + baselineURL + '?raw=true target="_blank">')
                outFile.write(img)
                outFile.write("\n</a>")
        outFile.write("\n")
    mdFile.close()
    outFile.close()

# Fill in the template parameters in a CMakeLists template file
# The output is a CMakeLists.txt file with Name substituted for {{{1}}}

def FillCMakeLists(S, Name, ExtraNames):
    r1 = re.sub(r'XXX', Name, S)
    reg = re.sub(r'YYY', ExtraNames, r1)
    return reg


def FillQtCMakeLists(S, Name):
    reg = re.sub(r'XXX', Name, S)
    return reg

#####################################################################

# Check for args
if len(sys.argv) < 4:
    displayHelp()
    exit(0)

# Get Repo, docs dir  and Wiki
RepoDir = sys.argv[1]
DocDir = sys.argv[2]
RepoURL = sys.argv[3]

# Initialize counts
cacheHits = 0
cacheMisses = 0

cxxCount = 0
csCount = 0
pyCount = 0
javaCount = 0

thumbCount = 0
doxyCount = 0

# Make sure the wiki docs folder exists
if not os.path.exists("docs"):
    os.makedirs("docs")

# Baseline top level
BaselineName = RepoDir + "/Testing/Baseline"

# Load the TinyUrl cache
CacheFile = RepoDir + "/Admin/TinyUrlCache"
CacheDict = dict()
LoadTinyUrlCache(CacheFile, CacheDict)

# Read the Wiki Templates
VTKTemplateFile = open(RepoDir + "/Admin/VTKCMakeLists", 'r')
VTKTemplate = VTKTemplateFile.read()
VTKTemplateFile.close()

VTKQtTemplateFile = open(RepoDir + "/Admin/VTKQtCMakeLists", 'r')
VTKQtTemplate = VTKQtTemplateFile.read()
VTKQtTemplateFile.close()

# Make the reference mtime to be the most recent of the two CMakeLists templates
refStat1 = os.stat(RepoDir + "/Admin/VTKQtCMakeLists")
refStat2 = os.stat(RepoDir + "/Admin/VTKCMakeLists")
refMtime = refStat1.st_mtime
if refStat2.st_mtime > refStat1.st_mtime:
    refMtime = refStat2.st_mtime

# Create a dict to hold code name and page name
codeToPage = dict()

# Create a dict to hold CMakeLists.txt file
exampleToCMake = dict()

# Create a dict to hold the file names for each example
exampleToFileNames = dict()

# Create Snippets directories for Cxx, Python and Java
if not os.path.exists(DocDir + "/Cxx/Snippets"):
    os.makedirs(DocDir + "/Cxx/Snippets")

if not os.path.exists(DocDir + "/Python/Snippets"):
    os.makedirs(DocDir + "/Python/Snippets")

if not os.path.exists(DocDir + "/Java/Snippets"):
    os.makedirs(DocDir + "/Java/Snippets")

# Add thumbnails and language links to each of the language summary pages, Snippets and Book figures
AddThumbnailsAndLanguageLinks(RepoDir, RepoURL, RepoDir + "/Cxx.md", DocDir + "/Cxx.md")
AddThumbnailsAndLanguageLinks(RepoDir, RepoURL, RepoDir + "/Python.md", DocDir + "/Python.md")
AddThumbnailsAndLanguageLinks(RepoDir, RepoURL, RepoDir + "/CSharp.md", DocDir + "/CSharp.md")
AddThumbnailsAndLanguageLinks(RepoDir, RepoURL, RepoDir + "/Java.md", DocDir + "/Java.md")

AddThumbnailsAndLanguageLinks(RepoDir, RepoURL, RepoDir + "/Cxx/Snippets.md", DocDir + "/Cxx/Snippets.md")
AddThumbnailsAndLanguageLinks(RepoDir, RepoURL, RepoDir + "/Python/Snippets.md", DocDir + "/Python/Snippets.md")
AddThumbnailsAndLanguageLinks(RepoDir, RepoURL, RepoDir + "/Java/Snippets.md", DocDir + "/Java/Snippets.md")
AddThumbnailsAndLanguageLinks(RepoDir, RepoURL, RepoDir + "/VTKBookFigures.md", DocDir + "/VTKBookFigures.md")
AddThumbnailsAndLanguageLinks(RepoDir, RepoURL, RepoDir + "/VTKFileFormats.md", DocDir + "/VTKFileFormats.md")

# C++ Snippets
files = os.listdir(RepoDir + "/Cxx/Snippets")
for f in files:
    snippet = os.path.splitext(f)[0]
    CopyFile(RepoDir + '/Cxx/Snippets/' + snippet + '.md', DocDir + '/Cxx/Snippets/' + snippet + '.md')

# Python Snippets
files = os.listdir(RepoDir + "/Python/Snippets")
for f in files:
    snippet = os.path.splitext(f)[0]
    CopyFile(RepoDir + '/Python/Snippets/' + snippet + '.md', DocDir + '/Python/Snippets/' + snippet + '.md')

# Java Snippets
files = os.listdir(RepoDir + "/Java/Snippets")
for f in files:
    snippet = os.path.splitext(f)[0]
    CopyFile(RepoDir + '/Java/Snippets/' + snippet + '.md', DocDir + '/Java/Snippets/' + snippet + '.md')

# Copy favicon.png
if not os.path.exists(DocDir + "/assets"):
    os.makedirs(DocDir + "/assets")
if not os.path.exists(DocDir + "/assets/images"):
    os.makedirs(DocDir + "/assets/images")
CopyFile(RepoDir + "/Images/favicon.png", DocDir + "/assets/images/favicon.png")

# Copy repo .md files
CopyFile(RepoDir + "/index.md", DocDir + "/index.md")
CopyFile(RepoDir + "/VTKBook.md", DocDir + "/VTKBook.md")

# Copy coverage files
if not os.path.exists(DocDir + "/Coverage"):
    os.makedirs(DocDir + "/Coverage")

CopyFile(RepoDir + "/Coverage/CxxVTKClassesNotUsed.md", DocDir + "/Coverage/CxxVTKClassesNotUsed.md")
CopyFile(RepoDir + "/Coverage/CxxVTKClassesUsed.md", DocDir + "/Coverage/CxxVTKClassesUsed.md")
CopyFile(RepoDir + "/Coverage/PythonVTKClassesNotUsed.md", DocDir + "/Coverage/PythonVTKClassesNotUsed.md")
CopyFile(RepoDir + "/Coverage/PythonVTKClassesUsed.md", DocDir + "/Coverage/PythonVTKClassesUsed.md")
CopyFile(RepoDir + "/Coverage/JavaVTKClassesNotUsed.md", DocDir + "/Coverage/JavaVTKClassesNotUsed.md")
CopyFile(RepoDir + "/Coverage/JavaVTKClassesUsed.md", DocDir + "/Coverage/JavaVTKClassesUsed.md")
CopyFile(RepoDir + "/Coverage/CSharpVTKClassesNotUsed.md", DocDir + "/Coverage/CSharpVTKClassesNotUsed.md")
CopyFile(RepoDir + "/Coverage/CSharpVTKClassesUsed.md", DocDir + "/Coverage/CSharpVTKClassesUsed.md")

# Copy Instructions files
if not os.path.exists(DocDir + "/Instructions"):
    os.makedirs(DocDir + "/Instructions")
CopyFile(RepoDir + "/Instructions/ForUsers.md", DocDir + "/Instructions/ForUsers.md")
CopyFile(RepoDir + "/Instructions/ForDevelopers.md", DocDir + "/Instructions/ForDevelopers.md")
CopyFile(RepoDir + "/Instructions/ForAdministrators.md", DocDir + "/Instructions/ForAdministrators.md")
CopyFile(RepoDir + "/Instructions/Guidelines.md", DocDir + "/Instructions/Guidelines.md")
CopyFile(RepoDir + "/Instructions/ConvertingFiguresToExamples.md", DocDir + "/Instructions/ConvertingFiguresToExamples.md")

# Copy VTKBook files
if not os.path.exists(DocDir + "/VTKBook"):
    os.makedirs(DocDir + "/VTKBook")
htmlIdSet = set()
CreateHtmlIds(RepoDir + "/VTKBook/00Preface.md")
CreateHtmlIds(RepoDir + "/VTKBook/01Chapter1.md")
CreateHtmlIds(RepoDir + "/VTKBook/02Chapter2.md")
CreateHtmlIds(RepoDir + "/VTKBook/03Chapter3.md")
CreateHtmlIds(RepoDir + "/VTKBook/04Chapter4.md")
CreateHtmlIds(RepoDir + "/VTKBook/05Chapter5.md")
CreateHtmlIds(RepoDir + "/VTKBook/06Chapter6.md")
CreateHtmlIds(RepoDir + "/VTKBook/07Chapter7.md")
CreateHtmlIds(RepoDir + "/VTKBook/08Chapter8.md")
CreateHtmlIds(RepoDir + "/VTKBook/09Chapter9.md")
CreateHtmlIds(RepoDir + "/VTKBook/10Chapter10.md")
CreateHtmlIds(RepoDir + "/VTKBook/11Chapter11.md")
CreateHtmlIds(RepoDir + "/VTKBook/12Chapter12.md")
CreateHtmlIds(RepoDir + "/VTKBook/13Glossary.md")

print("Found ", len(htmlIdSet), " figures with html ids")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/00Preface.md", DocDir  + "/VTKBook/00Preface.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/01Chapter1.md", DocDir + "/VTKBook/01Chapter1.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/02Chapter2.md", DocDir + "/VTKBook/02Chapter2.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/03Chapter3.md", DocDir + "/VTKBook/03Chapter3.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/04Chapter4.md", DocDir + "/VTKBook/04Chapter4.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/05Chapter5.md", DocDir + "/VTKBook/05Chapter5.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/06Chapter6.md", DocDir + "/VTKBook/06Chapter6.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/07Chapter7.md", DocDir + "/VTKBook/07Chapter7.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/08Chapter8.md", DocDir + "/VTKBook/08Chapter8.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/09Chapter9.md", DocDir + "/VTKBook/09Chapter9.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/10Chapter10.md", DocDir + "/VTKBook/10Chapter10.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/11Chapter11.md", DocDir + "/VTKBook/11Chapter11.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/12Chapter12.md", DocDir + "/VTKBook/12Chapter12.md")
CopyChapterAndAddLinks(RepoDir + "/VTKBook/13Glossary.md", DocDir + "/VTKBook/13Glossary.md")

# Copy VTKBookLaTeX files
if not os.path.exists(DocDir + "/VTKBookLaTeX"):
    os.makedirs(DocDir + "/VTKBookLaTeX")
CopyFile(RepoDir + "/VTKBookLaTeX/VTKTextBook.md", DocDir + "/VTKBookLaTeX/VTKTextBook.md")

# Scan all Cxx directories and look for extras
allExtras = set()
for root, dirs, files in os.walk(RepoDir):
    to_find = RepoDir + "/" + "Cxx"
    start = root.find(to_find)
    if start < 0:
        continue
    for f in files:
        fileName = os.path.splitext(f)
        if fileName[1] == ".extras":
            ExtrasPath = root + "/" + f
            ExtrasFile = open(ExtrasPath, 'r')
            for line in ExtrasFile:
                line = line.strip()
                allExtras.add(line)
            ExtrasFile.close()

# Get list of all  examples
Languages = dict()
Languages["Cxx"] = ".cxx"
Languages["Python"] = ".py"
Languages["Java"] = ".java"
Languages["CSharp"] = ".cs"

for lang, langExt in Languages.items():
    for root, dirs, files in os.walk(RepoDir):
        to_find = RepoDir + "/" + lang
        start = root.find(to_find)
        if start < 0:
            continue
        # Get the part of the file name that comes after RepoDir
        # e.g. if the file name is VTKExamples/Cxx/GeometricObjects/Line,
        # Path will be Cxx/GeometriObjects/Line
        KitName = root[start + 1 + len(to_find):]
        if KitName.find("Boneyard") >= 0:
            continue
        if KitName.find("Broken") >= 0:
            continue
        for f in files:
            otherLanguages = list()
            # skip files that are listed as extras
            if f in allExtras:
                continue
            if f == "CMakeLists.txt":
                continue
            ExampleName = os.path.splitext(f)[0]
            ExampleExt = os.path.splitext(f)[1]
            if ExampleExt != langExt:
                continue
            # Find examples in other languages
            fp = root + "/" + f
            for l, lExt in Languages.items():
                otherLink = FindOtherGivenLang("", os.path.splitext(fp)[0], lang, l, lExt)
                if otherLink != '' :
                    otherLink = re.sub(r'src',r'', otherLink)
                    otherLanguages.append(otherLink)
#           otherLink = FindOtherGivenLang("", os.path.splitext(fp)[0], lang, "Cxx", ".cxx")
#            if otherLink != '' :
#                otherLink = re.sub(r'src',r'', otherLink)
#                otherLanguages.append(otherLink)
#            otherLink = FindOtherGivenLang("", os.path.splitext(fp)[0], lang, "Python", ".py")
#            if otherLink != '' :
#                otherLink = re.sub(r'src',r'', otherLink)
#                otherLanguages.append(otherLink)
#            otherLink = FindOtherGivenLang("", os.path.splitext(fp)[0], lang, "Java", ".java")
#            if otherLink != '' :
#                otherLink = re.sub(r'src',r'', otherLink)
#                otherLanguages.append(otherLink)
#            otherLink = FindOtherGivenLang("", os.path.splitext(fp)[0], lang, "CSharp", ".cs")
#            if otherLink != '' :
#                otherLink = re.sub(r'src',r'', otherLink)
#                otherLanguages.append(otherLink)
            BaselinePath = RepoDir + "/Testing/Baseline/" + lang + "/" + KitName + "/Test" + ExampleName + ".png"
            PathName = DocDir + "/" + lang + "/" + KitName
            if not os.path.exists(PathName):
                if PathName != "":
                    os.makedirs(PathName)
            OutputFile = DocDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".md"
            MdFile = open(OutputFile, 'w')
            MdFile.write("[VTKExamples](/)/[" + lang + "](/" + lang + ")/" + KitName + "/" + ExampleName + "\n\n")

            if os.path.isfile(BaselinePath):
                ImgUrl = RepoURL + "/blob/master/src/Testing/Baseline/" + lang + "/" + KitName + "/Test" + ExampleName + ".png?raw=true"
                # href to open image in new tab
                MdFile.write('<a href="' + ImgUrl + ' target="_blank">' + '\n')
                MdFile.write('<img style="border:2px solid beige;float:center" src="' + ImgUrl + '" width="256" />' + '\n')
                MdFile.write("</a>" + "\n")
                MdFile.write("<hr>\n")
                MdFile.write("\n")
            DescriptionPath = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".md"
            # Add a description if a .md file exists for the example
            if os.path.isfile(DescriptionPath):
                DescriptionFile = open(RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".md", 'r')
                description = DescriptionFile.read()
                DescriptionFile.close()
                description = AddDoxygen(description)
                MdFile.write(description)
            # Add examples from other languages if they exist
            if len(otherLanguages) > 0 :
                seeAlso = '\n!!! Tip "Other Languages"\n'
                see = "    See "
                for other in otherLanguages :
                    seeAlso += see + other
                    see = ", "
                seeAlso += "\n"
                MdFile.write(seeAlso)
                MdFile.write("\n")

            SrcFileName = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + langExt
            SrcFile = open(SrcFileName, 'r')
            src = SrcFile.read()
            SrcFile.close()
            hiliteLines = FindLinesWithVTK(SrcFileName)
            MdFile.write("###Code\n")
            MdFile.write("**" + ExampleName + langExt + "**" + "\n")
            if langExt == ".cxx":
                MdFile.write("``` c++ " + hiliteLines + "\n")
                cxxCount = cxxCount + 1
            elif langExt == ".cs":
                MdFile.write("```csharp" + hiliteLines + "\n")
                csCount = csCount + 1
            elif langExt == ".py":
                MdFile.write("```python" + hiliteLines + "\n")
                pyCount = pyCount + 1
            elif langExt == ".java":
                MdFile.write("```java" + hiliteLines + "\n")
                javaCount = javaCount + 1
            MdFile.write(src)
            MdFile.write("```" + "\n")

            # Store the full file names for the example
            if ExampleName not in exampleToFileNames:
                exampleToFileNames[ExampleName] = set()
            SrcFile = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ExampleExt
            exampleToFileNames[ExampleName].add(SrcFile)

            ExtrasPath = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".extras"
            ExtraNames = ""
            if os.path.isfile(ExtrasPath):
                ExtrasFile = open(ExtrasPath, 'r')
                for line in ExtrasFile:
                    line = line.strip()
                    if line == '':
                        continue
                    ExtraPath = RepoDir + "/" + lang + "/" + KitName + "/" + line
                    SrcFile = RepoDir + "/" + lang + "/" + KitName + "/" + line

                    exampleToFileNames[ExampleName].add(SrcFile)
                    ExtraFile = open(ExtraPath, 'r')
                    extraCode = ExtraFile.read()
                    ExtraFile.close()
                    WriteCxxCode(MdFile, line, extraCode)
                    extent = os.path.splitext(line)
                    if extent[1] == ".cxx":
                        ExtraNames += " " + line
                ExtrasFile.close()
            CustomCMakePath = RepoDir + "/" + lang + "/" + KitName + "/" + ExampleName + ".cmake"
            if os.path.isfile(CustomCMakePath):
                CustomCMakeFile = open(CustomCMakePath, 'r')
                cmake = CustomCMakeFile.read()
                CustomCMakeFile.close()
            else:
                if IsQtExample(src):
                    CMakeFile = open(RepoDir + "/Admin/VTKQtCMakeLists", 'r')
                    CMakeContents = CMakeFile.read()
                    CMakeFile.close()
                    cmake = FillQtCMakeLists(CMakeContents, ExampleName)
                else:
                    CMakeFile = open(RepoDir + "/Admin/VTKCMakeLists", 'r')
                    CMakeContents = CMakeFile.read()
                    CMakeFile.close()
                    cmake = FillCMakeLists(CMakeContents, ExampleName, ExtraNames)
            if lang == "Cxx":
                exampleToCMake[ExampleName] = GetVTKCMakelists(cmake)
                MdFile.write(cmake)
            MdFile.close()
            codeToPage[ExampleName + langExt] = "/" + lang + "/" + KitName + "/" + ExampleName

# Generate an html page that links each example code file to its Wiki Example
# page
indexFile = open(DocDir + "/ExampleCodeToWikiPage.html", 'w')
indexFile.write("Navigate to the page that contains the source code of an example<br>")
indexFile.write("\n")
sortedByCode = sorted(codeToPage.items())
for item in sortedByCode:
    indexFile.write("<A HREF=" + RepoURL + "/wikis" + re.sub(" ", "_", item[1]) + ">" + item[0] + "</A>")
    indexFile.write("<A HREF=" + RepoURL + "/blob/master" + re.sub(" ", "_", item[1]) + ".md" + ">" + "(md)" + "</A>")
    indexFile.write("<br>\n")
indexFile.close()

# Create tarballs for each example
tmpDir = tempfile.mkdtemp(prefix="VTKTarballs") + "/"

# Create the Tarballs directory in the source tree if not present
# If it does not exist, assume the tarball repo has not been cloned
# and we need to ignore tar files
if not os.path.exists("src/Tarballs"):
    os.makedirs("src/Tarballs")
    ignoreFile = open("src/Tarballs/.gitignore", 'w')
    ignoreFile.write("*,tar\n")
    ignoreFile.close()

# Create tarballs
# For each example page, create a directory and copy that example's files
# into the directory
# If the example has a CMakeLists.txt file, copy that.
# Also, create a subdir called build. This directory is handy when you want to
# configure with CMake and build the example.
for example in exampleToFileNames:
    if example not in exampleToCMake:
        continue
    # Make the directories for the example
    srcDir = tmpDir + example
    codeFileName = srcDir + "/" + example + ".cxx"
    if not os.path.exists(srcDir):
        os.makedirs(srcDir)

        # An example may have multiple source files
        for exampleFileName in exampleToFileNames[example]:
            # Get the file name
            tarFileName = srcDir + "/" + os.path.split(exampleFileName)[1]
            # Copy the code for the example
            shutil.copy(exampleFileName, tarFileName)
            os.utime(tarFileName, (refMtime, refMtime))

    # Some examples do not have a CMakeLists.txt file
    if example in exampleToCMake:
        os.makedirs(srcDir + "/build")
        cmakeFileName = srcDir + "/" + "CMakeLists.txt"
        cmakeFile = open(cmakeFileName, 'w')
        cmakeFile.write(exampleToCMake[example][0])
        cmakeFile.close()
        os.utime(cmakeFileName, (refMtime, refMtime))

    # Set the mtimes for the directories containing the example
    # Since the mtime is stored in the tar header for each file and directory,
    # we need a consistent time so that a tar of an unchanged example will be equal
    # to the one in the repo
    os.utime(srcDir, (refMtime, refMtime))
    os.utime(srcDir + "/build", (refMtime, refMtime))

    # Now create the tar file for the example
    # The tarballs are stored in the source tree
    tar = tarfile.open("src/Tarballs/" + example + ".tar", "w")
    tar.add(srcDir, arcname=example)
    tar.close()

os.utime(tmpDir, (0, refMtime))
# Cleanup the temporary directories
shutil.rmtree(tmpDir)


# Update the cache file if necessary
if cacheMisses > 0:
    cf = open(CacheFile, 'w')
    for key in CacheDict:
        cf.write(key + " " + CacheDict[key] + "\n")
    cf.close()

# Report stats
print("ScrapeRepo Summary")
print("    C++ examples: " + str(cxxCount))
print("    CSharp examples: " + str(csCount))
print("    Python examples: " + str(pyCount))
print("    Java examples: " + str(javaCount))
print("    Total examples: " + str(cxxCount + csCount + pyCount + javaCount))
print("    Doxygen added: " + str(doxyCount))
print("    Thumbnails added: " + str(thumbCount))
print("    TinyUrl Cache hits: " + str(cacheHits))
print("    TinyUrl Cache misses: " + str(cacheMisses))
