pandoc-reader/pandoc_reader.py

"""Reader that processes Pandoc Markdown and returns HTML5."""
import json
import math
import os
import shutil
import subprocess

import bs4
from ruamel.yaml import YAML, constructor

from pelican import signals
from pelican.readers import BaseReader
from pelican.utils import pelican_open

DEFAULT_READING_SPEED = 200  # Words per minute
DEFAULT_PANDOC_EXECUTABLE = "pandoc"
DIR_PATH = os.path.dirname(__file__)
ENCODED_LINKS_TO_RAW_LINKS_MAP = {
    "%7Bstatic%7D": "{static}",
    "%7Battach%7D": "{attach}",
    "%7Bfilename%7D": "{filename}",
}
FILE_EXTENSIONS = ["md", "mkd", "mkdn", "mdwn", "mdown", "markdown", "Rmd"]
PANDOC_READER_HTML_TEMPLATE = "pandoc-reader-default.html"
PANDOC_SUPPORTED_MAJOR_VERSION = 2
PANDOC_SUPPORTED_MINOR_VERSION = 11

TEMPLATES_PATH = os.path.abspath(os.path.join(DIR_PATH, "templates"))
UNSUPPORTED_ARGUMENTS = ("--standalone", "--self-contained")
VALID_BIB_EXTENSIONS = ["json", "yaml", "bibtex", "bib"]

# Markdown variants supported in defaults files
# Update as Pandoc adds or removes support for formats
VALID_INPUT_FORMATS = (
    "commonmark",
    "commonmark_x",
    "gfm",
    "markdown",
    "markdown_mmd",
    "markdown_phpextra",
    "markdown_strict",
)
VALID_OUTPUT_FORMATS = ("html", "html5")


class PandocReader(BaseReader):
    """Convert files written in Pandoc Markdown to HTML 5."""

    enabled = True
    file_extensions = FILE_EXTENSIONS

    def read(self, source_path):
        """Parse Pandoc Markdown and return HTML5 markup and metadata."""
        # Get the user-defined path to the Pandoc executable or fall back to default
        pandoc_executable = self.settings.get(
            "PANDOC_EXECUTABLE_PATH", DEFAULT_PANDOC_EXECUTABLE
        )

        # If user-defined path, expand and make it absolute in case the path is relative
        if pandoc_executable != DEFAULT_PANDOC_EXECUTABLE:
            pandoc_executable = os.path.abspath(os.path.expanduser(pandoc_executable))

        # Check if pandoc is installed and is executable
        if not shutil.which(pandoc_executable):
            raise Exception("Could not find Pandoc. Please install.")

        # Check if the version of pandoc installed is 2.11 or higher
        self._check_pandoc_version(pandoc_executable)

        # Open Markdown file and read content
        content = ""
        with pelican_open(source_path) as file_content:
            content = file_content

        # Retrieve HTML content and metadata
        output, metadata = self._create_html(source_path, content, pandoc_executable)

        return output, metadata

    def _create_html(self, source_path, content, pandoc_executable):
        """Create HTML5 content."""
        # Get settings set in pelicanconf.py
        defaults_files = self.settings.get("PANDOC_DEFAULTS_FILES", [])

        # Adding support for the old defaults file setting
        # which will be removed in future versions of this plugin
        if self.settings.get("PANDOC_DEFAULT_FILES", []):
            defaults_files = self.settings.get("PANDOC_DEFAULT_FILES")

        arguments = self.settings.get("PANDOC_ARGS", [])
        extensions = self.settings.get("PANDOC_EXTENSIONS", [])

        if isinstance(extensions, list):
            extensions = "".join(extensions)

        # Check if source content has a YAML metadata block
        self._check_yaml_metadata_block(content)

        # Check validity of arguments or defaults files
        table_of_contents, citations = self._validate_fields(
            defaults_files, arguments, extensions
        )

        # Construct preliminary pandoc command
        pandoc_cmd = self._construct_pandoc_command(
            pandoc_executable, defaults_files, arguments, extensions
        )

        # Find and add bibliography if citations are specified
        if citations:
            for bib_file in self._find_bibs(source_path):
                pandoc_cmd.append("--bibliography={0}".format(bib_file))

        # Create HTML content using pandoc-reader-default.html template
        output = self._run_pandoc(pandoc_cmd, content)

        # Extract table of contents, text and metadata from HTML output
        output, toc, pandoc_metadata = self._extract_contents(output, table_of_contents)

        # Replace all occurrences of %7Bstatic%7D to {static},
        # %7Battach%7D to {attach} and %7Bfilename%7D to {filename}
        # so that static links are resolvable by pelican
        for encoded_str, raw_str in ENCODED_LINKS_TO_RAW_LINKS_MAP.items():
            output = output.replace(encoded_str, raw_str)

        # Parse Pandoc metadata and add it to Pelican
        metadata = self._process_metadata(pandoc_metadata)

        if table_of_contents:
            # Create table of contents and add to metadata
            metadata["toc"] = self.process_metadata("toc", toc)

        return output, metadata

    def _validate_fields(self, defaults_files, arguments, extensions):
        """Validate fields and return citations and ToC request values."""
        # If defaults_files is empty then validate the argument and extensions
        if not defaults_files:
            # Validate the arguments to see that they are supported
            # by the plugin
            self._check_arguments(arguments)

            # Check if citations have been requested
            citations = self._check_if_citations(arguments, extensions)

            # Check if table of contents has been requested
            table_of_contents = self._check_if_toc(arguments)
        else:
            # Validate defaults files and get the citations
            # abd table of contents request value
            citations, table_of_contents = self._check_defaults(defaults_files)
        return table_of_contents, citations

    def _check_defaults(self, defaults_files):
        """Check if the given Pandoc defaults file has valid values."""
        citations = False
        table_of_contents = False

        # Get the data in all defaults files as a string
        defaults_data = ""
        for defaults_file in defaults_files:
            with open(defaults_file, "r") as file_handle:
                for line in file_handle.readlines():
                    defaults_data += line

        # Convert YAML data to a Python dictionary
        defaults = {}
        try:
            yaml = YAML()
            defaults = yaml.load(defaults_data)
        except constructor.DuplicateKeyError as duplicate_key_error:
            raise ValueError(
                "Duplicate keys defined in multiple defaults files."
            ) from duplicate_key_error

        self._check_if_unsupported_settings(defaults)
        reader = self._check_input_format(defaults)
        self._check_output_format(defaults)

        if not citations:
            citeproc_specified = False

            # Cases where citeproc is specified as citeproc: true
            if defaults.get("citeproc", ""):
                citeproc_specified = True

            # Cases where citeproc is specified in filters
            elif "citeproc" in defaults.get("filters", ""):
                citeproc_specified = True

            # The extension +citations is enabled by default in Pandoc 2.11
            # we are checking that the extension is not disabled using -citations
            if citeproc_specified and "-citations" not in reader:
                citations = True

        if not table_of_contents:
            if defaults.get("table-of-contents", ""):
                table_of_contents = True

        return citations, table_of_contents

    def _process_metadata(self, pandoc_metadata):
        """Process Pandoc metadata and add it to Pelican."""
        # Cycle through the metadata and process them
        metadata = {}
        for key, value in pandoc_metadata.items():
            key = key.lower()
            if value and isinstance(value, str):
                value = value.strip().strip('"')

            # Process the metadata
            metadata[key] = self.process_metadata(key, value)
        return metadata

    @staticmethod
    def _check_pandoc_version(pandoc_executable):
        """Check that the specified version of Pandoc is 2.11 or higher."""
        output = subprocess.run(
            [pandoc_executable, "--version"],
            capture_output=True,
            encoding="utf-8",
            check=True,
        )

        # Returns a string of the form pandoc <version>
        pandoc_version = output.stdout.split("\n")[0]

        # Get the major and minor version from the above version string
        major_version = pandoc_version.split()[1].split(".")[0]
        minor_version = pandoc_version.split()[1].split(".")[1]

        # Pandoc major version less than 2 are not supported
        if int(major_version) < PANDOC_SUPPORTED_MAJOR_VERSION:
            raise Exception("Pandoc version must be 2.11 or higher.")

        # Pandoc major version 2 minor version less than 11 are not supported
        if (
            int(major_version) == PANDOC_SUPPORTED_MAJOR_VERSION
            and int(minor_version) < PANDOC_SUPPORTED_MINOR_VERSION
        ):
            raise Exception("Pandoc version must be 2.11 or higher.")

    @staticmethod
    def _check_yaml_metadata_block(content):
        """Check if the source content has a YAML metadata block."""
        # Check that the given content is not empty
        if not content:
            raise Exception("Could not find metadata. File is empty.")

        # Split content into a list of lines
        content_lines = content.splitlines()

        # Check that the first line of the file starts with a YAML block
        if content_lines[0].rstrip() not in ["---"]:
            raise Exception("Could not find metadata header '---'.")

        # Find the end of the YAML block
        yaml_block_end = ""
        for line_num, line in enumerate(content_lines[1:]):
            if line.rstrip() in ["---", "..."]:
                yaml_block_end = line_num
                break

        # Check if the end of the YAML block was found
        if not yaml_block_end:
            raise Exception("Could not find end of metadata block.")

    @staticmethod
    def _construct_pandoc_command(
        pandoc_executable, defaults_files, arguments, extensions
    ):
        """Construct Pandoc command for content."""
        pandoc_cmd = [
            pandoc_executable,
            "--standalone",
            "--template={}".format(
                os.path.join(TEMPLATES_PATH, PANDOC_READER_HTML_TEMPLATE)
            ),
        ]
        if not defaults_files:
            pandoc_cmd.extend(["--from", "markdown" + extensions, "--to", "html5"])
            pandoc_cmd.extend(arguments)
        else:
            for defaults_file in defaults_files:
                pandoc_cmd.append("--defaults={0}".format(defaults_file))
        return pandoc_cmd

    @staticmethod
    def _run_pandoc(pandoc_cmd, content):
        """Execute the given pandoc command and return output."""
        output = subprocess.run(
            pandoc_cmd,
            input=content,
            capture_output=True,
            encoding="utf-8",
            check=True,
        )
        return output.stdout

    @staticmethod
    def _extract_contents(html_output, table_of_contents):
        """Extract body html, table of contents and metadata from output."""
        # Extract pandoc metadata from html output
        pandoc_json_metadata, _, html_output = html_output.partition("\n")

        # Convert JSON string to dict
        pandoc_metadata = json.loads(pandoc_json_metadata)

        # Parse HTML output
        soup = bs4.BeautifulSoup(html_output, "html.parser")

        # Extract the table of contents if one was requested
        toc = ""
        if table_of_contents:
            # Find the table of contents
            toc = soup.body.find("nav", id="TOC")

            if toc:
                # Convert it to a string
                toc = str(toc)

                # Replace id=TOC with class="toc"
                toc = toc.replace('id="TOC"', 'class="toc"')

                # Remove the table of contents from the HTML output
                soup.body.find("nav", id="TOC").decompose()

        # Remove body tag around html output
        soup.body.unwrap()

        # Strip leading and trailing spaces
        html_output = str(soup).strip()

        return html_output, toc, pandoc_metadata

    @staticmethod
    def _check_if_citations(arguments, extensions):
        """Check if citations are specified."""
        citations = False
        if arguments and extensions:
            # The +citations extension is enabled by default in Pandoc 2.11
            # therefore we do a check to see that it is not disabled in extensions
            if (
                "--citeproc" in arguments or "-C" in arguments
            ) and "-citations" not in extensions:
                citations = True
        return citations

    @staticmethod
    def _check_if_toc(arguments):
        """Check if a table of contents should be generated."""
        table_of_contents = False
        if arguments:
            if "--toc" in arguments or "--table-of-contents" in arguments:
                table_of_contents = True
        return table_of_contents

    @staticmethod
    def _find_bibs(source_path):
        """Find bibliographies recursively in the sourcepath given."""
        bib_files = []
        filename = os.path.splitext(os.path.basename(source_path))[0]
        directory_path = os.path.dirname(os.path.abspath(source_path))
        for root, _, files in os.walk(directory_path):
            for extension in VALID_BIB_EXTENSIONS:
                bib_name = ".".join([filename, extension])
                if bib_name in files:
                    bib_files.append(os.path.join(root, bib_name))
        return bib_files

    @staticmethod
    def _check_arguments(arguments):
        """Check to see that only supported arguments have been passed."""
        for arg in arguments:
            if arg in UNSUPPORTED_ARGUMENTS:
                raise ValueError("Argument {0} is not supported.".format(arg))

    @staticmethod
    def _check_if_unsupported_settings(defaults):
        """Check if unsupported settings are specified in the defaults."""
        for arg in UNSUPPORTED_ARGUMENTS:
            arg = arg[2:]
            if defaults.get(arg, ""):
                raise ValueError("The default {} should be set to false.".format(arg))

    @staticmethod
    def _check_input_format(defaults):
        """Check if the input format given is a Markdown variant."""
        reader = ""
        reader_input = defaults.get("reader", "")
        from_input = defaults.get("from", "")

        # Case where no input format is specified
        if not reader_input and not from_input:
            raise ValueError("No input format specified.")

        # Case where both reader and from are specified which is not supported
        if reader_input and from_input:
            raise ValueError(
                (
                    "Specifying both from and reader is not supported."
                    " Please specify just one."
                )
            )

        if reader_input or from_input:
            if reader_input:
                reader = reader_input
            elif from_input:
                reader = from_input

            reader_prefix = reader.replace("+", "-").split("-")[0]

            # Check to see if the reader_prefix matches a valid input format
            if reader_prefix not in VALID_INPUT_FORMATS:
                raise ValueError("Input type has to be a Markdown variant.")
        return reader

    @staticmethod
    def _check_output_format(defaults):
        """Check if the output format is HTML or HTML5."""
        writer_output = defaults.get("writer", "")
        to_output = defaults.get("to", "")

        # Case where both writer and to are specified which is not supported
        if writer_output and to_output:
            raise ValueError(
                (
                    "Specifying both to and writer is not supported."
                    " Please specify just one."
                )
            )

        # Case where neither writer nor to value is set to html
        if (
            writer_output not in VALID_OUTPUT_FORMATS
            and to_output not in VALID_OUTPUT_FORMATS
        ):
            output_formats = " or ".join(VALID_OUTPUT_FORMATS)
            raise ValueError(
                "Output format type must be either {}.".format(output_formats)
            )


def add_reader(readers):
    """Add the PandocReader as the reader for all Pandoc Markdown files."""
    for ext in PandocReader.file_extensions:
        readers.reader_classes[ext] = PandocReader


def register():
    """Register the PandocReader."""
    signals.readers_init.connect(add_reader)