tutorial-reconstruction-algorithms/bin/util.py

0001 import sys
0002 import os
0003 import json
0004 from subprocess import Popen, PIPE
0005
0006 # Import this way to produce a more useful error message.
0007 try:
0008     import yaml
0009 except ImportError:
0010     print('Unable to import YAML module: please install PyYAML', file=sys.stderr)
0011     sys.exit(1)
0012
0013 __all__ = ['check_unwanted_files', 'load_yaml', 'read_markdown', 'require']
0014
0015 # Files that shouldn't be present.
0016 UNWANTED_FILES = [
0017     '.nojekyll'
0018 ]
0019
0020 def read_markdown(parser, path):
0021     """
0022     Get YAML and AST for Markdown file, returning
0023     {'metadata':yaml, 'metadata_len':N, 'text':text, 'lines':[(i, line, len)], 'doc':doc}.
0024     """
0025
0026     # Split and extract YAML (if present).
0027     with open(path, 'r', encoding='utf-8') as reader:
0028         body = reader.read()
0029     metadata_raw, metadata_yaml, body = split_metadata(path, body)
0030
0031     # Split into lines.
0032     metadata_len = 0 if metadata_raw is None else metadata_raw.count('\n')
0033     lines = [(metadata_len+i+1, line, len(line))
0034              for (i, line) in enumerate(body.split('\n'))]
0035
0036     # Parse Markdown.
0037     cmd = 'bundle exec ruby {0}'.format(parser)
0038     p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE,
0039               close_fds=True, universal_newlines=True, encoding='utf-8')
0040     stdout_data, stderr_data = p.communicate(body)
0041     doc = json.loads(stdout_data)
0042
0043     return {
0044         'metadata': metadata_yaml,
0045         'metadata_len': metadata_len,
0046         'text': body,
0047         'lines': lines,
0048         'doc': doc
0049     }
0050
0051
0052 def split_metadata(path, text):
0053     """
0054     Get raw (text) metadata, metadata as YAML, and rest of body.
0055     If no metadata, return (None, None, body).
0056     """
0057
0058     metadata_raw = None
0059     metadata_yaml = None
0060
0061     pieces = text.split('---', 2)
0062     if len(pieces) == 3:
0063         metadata_raw = pieces[1]
0064         text = pieces[2]
0065         try:
0066             metadata_yaml = yaml.load(metadata_raw, Loader=yaml.SafeLoader)
0067         except yaml.YAMLError as e:
0068             message = 'Unable to parse YAML header in {0}:\n{1}'
0069             print(message.format(path, e), file=sys.stderr)
0070
0071     return metadata_raw, metadata_yaml, text
0072
0073
0074 def load_yaml(filename):
0075     """
0076     Wrapper around YAML loading so that 'import yaml' is only needed
0077     in one file.
0078     """
0079
0080     try:
0081         with open(filename, 'r', encoding='utf-8') as reader:
0082             return yaml.load(reader, Loader=yaml.SafeLoader)
0083     except yaml.YAMLError as e:
0084         message = 'ERROR: Unable to load YAML file {0}:\n{1}'
0085         print(message.format(filename, e), file=sys.stderr)
0086     except (FileNotFoundError, IOError):
0087         message = 'ERROR: File {} not found'
0088         print(message.format(filename), file=sys.stderr)
0089
0090     return {}
0091
0092 def check_unwanted_files(dir_path, reporter):
0093     """
0094     Check that unwanted files are not present.
0095     """
0096
0097     for filename in UNWANTED_FILES:
0098         path = os.path.join(dir_path, filename)
0099         reporter.check(not os.path.exists(path),
0100                        path,
0101                        "Unwanted file found")
0102
0103
0104 def require(condition, message, fatal=False):
0105     """Fail if condition not met."""
0106
0107     if not condition:
0108         print(message, file=sys.stderr)
0109
0110         if fatal:
0111             sys.exit(1)