first commit

This commit is contained in:
Ayxan
2022-05-23 00:16:32 +04:00
commit d660f2a4ca
24786 changed files with 4428337 additions and 0 deletions

View File

@@ -0,0 +1,311 @@
import hashlib
import os
import json
import random
import collections
from operator import itemgetter
import warnings
import jinja2
from docutils import nodes
from docutils.statemachine import ViewList
from docutils.parsers.rst import Directive
from docutils.parsers.rst.directives import flag
from sphinx.util.nodes import nested_parse_with_titles
from .utils import (
get_docstring_and_rest,
prev_this_next,
create_thumbnail,
create_generic_image,
)
from altair.utils.execeval import eval_block
from altair.examples import iter_examples
EXAMPLE_MODULE = "altair.examples"
GALLERY_TEMPLATE = jinja2.Template(
"""
.. This document is auto-generated by the altair-gallery extension. Do not modify directly.
.. _{{ gallery_ref }}:
{{ title }}
{% for char in title %}-{% endfor %}
This gallery contains a selection of examples of the plots Altair can create.
Some may seem fairly complicated at first glance, but they are built by combining a simple set of declarative building blocks.
Many draw upon sample datasets compiled by the `Vega <https://vega.github.io/vega/>`_ project. To access them yourself, install `vega_datasets <https://github.com/altair-viz/vega_datasets>`_.
.. code-block:: none
$ pip install vega_datasets
{% for grouper, group in examples %}
.. _gallery-category-{{ grouper }}:
{{ grouper }}
{% for char in grouper %}~{% endfor %}
.. raw:: html
<span class="gallery">
{% for example in group %}
<a class="imagegroup" href="{{ example.name }}.html">
<span class="image" alt="{{ example.title }}" style="background-image: url({{ image_dir }}/{{ example.name }}-thumb.png);"></span>
<span class="image-title">{{ example.title }}</span>
</a>
{% endfor %}
</span>
<div style='clear:both;'></div>
.. toctree::
:hidden:
{% for example in group %}
{{ example.name }}
{%- endfor %}
{% endfor %}
"""
)
MINIGALLERY_TEMPLATE = jinja2.Template(
"""
.. raw:: html
<div id="showcase">
<div class="examples">
{% for example in examples %}
<a class="preview" href="{{ gallery_dir }}/{{ example.name }}.html" style="background-image: url({{ image_dir }}/{{ example.name }}-thumb.png)"></a>
{% endfor %}
</div>
</div>
"""
)
EXAMPLE_TEMPLATE = jinja2.Template(
"""
.. This document is auto-generated by the altair-gallery extension. Do not modify directly.
.. _gallery_{{ name }}:
{{ docstring }}
.. altair-plot::
{% if code_below %}:code-below:{% endif %}
{{ code | indent(4) }}
.. toctree::
:hidden:
"""
)
def save_example_pngs(examples, image_dir, make_thumbnails=True):
"""Save example pngs and (optionally) thumbnails"""
if not os.path.exists(image_dir):
os.makedirs(image_dir)
# store hashes so that we know whether images need to be generated
hash_file = os.path.join(image_dir, "_image_hashes.json")
if os.path.exists(hash_file):
with open(hash_file) as f:
hashes = json.load(f)
else:
hashes = {}
for example in examples:
filename = example["name"] + ".png"
image_file = os.path.join(image_dir, filename)
example_hash = hashlib.md5(example["code"].encode()).hexdigest()
hashes_match = hashes.get(filename, "") == example_hash
if hashes_match and os.path.exists(image_file):
print("-> using cached {}".format(image_file))
else:
# the file changed or the image file does not exist. Generate it.
print("-> saving {}".format(image_file))
chart = eval_block(example["code"])
try:
chart.save(image_file)
hashes[filename] = example_hash
except ImportError:
warnings.warn("Unable to save image: using generic image")
create_generic_image(image_file)
with open(hash_file, "w") as f:
json.dump(hashes, f)
if make_thumbnails:
params = example.get("galleryParameters", {})
thumb_file = os.path.join(image_dir, example["name"] + "-thumb.png")
create_thumbnail(image_file, thumb_file, **params)
# Save hashes so we know whether we need to re-generate plots
with open(hash_file, "w") as f:
json.dump(hashes, f)
def populate_examples(**kwds):
"""Iterate through Altair examples and extract code"""
examples = sorted(iter_examples(), key=itemgetter("name"))
for example in examples:
docstring, category, code, lineno = get_docstring_and_rest(example["filename"])
example.update(kwds)
if category is None:
category = "other charts"
example.update(
{
"docstring": docstring,
"title": docstring.strip().split("\n")[0],
"code": code,
"category": category.title(),
"lineno": lineno,
}
)
return examples
class AltairMiniGalleryDirective(Directive):
has_content = False
option_spec = {
"size": int,
"names": str,
"indices": lambda x: list(map(int, x.split())),
"shuffle": flag,
"seed": int,
"titles": bool,
"width": str,
}
def run(self):
size = self.options.get("size", 15)
names = [name.strip() for name in self.options.get("names", "").split(",")]
indices = self.options.get("indices", [])
shuffle = "shuffle" in self.options
seed = self.options.get("seed", 42)
titles = self.options.get("titles", False)
width = self.options.get("width", None)
env = self.state.document.settings.env
app = env.app
gallery_dir = app.builder.config.altair_gallery_dir
examples = populate_examples()
if names:
if len(names) < size:
raise ValueError(
"altair-minigallery: if names are specified, "
"the list must be at least as long as size."
)
mapping = {example["name"]: example for example in examples}
examples = [mapping[name] for name in names]
else:
if indices:
examples = [examples[i] for i in indices]
if shuffle:
random.seed(seed)
random.shuffle(examples)
if size:
examples = examples[:size]
include = MINIGALLERY_TEMPLATE.render(
image_dir="/_static",
gallery_dir=gallery_dir,
examples=examples,
titles=titles,
width=width,
)
# parse and return documentation
result = ViewList()
for line in include.split("\n"):
result.append(line, "<altair-minigallery>")
node = nodes.paragraph()
node.document = self.state.document
nested_parse_with_titles(self.state, result, node)
return node.children
def main(app):
gallery_dir = app.builder.config.altair_gallery_dir
target_dir = os.path.join(app.builder.srcdir, gallery_dir)
image_dir = os.path.join(app.builder.srcdir, "_images")
gallery_ref = app.builder.config.altair_gallery_ref
gallery_title = app.builder.config.altair_gallery_title
examples = populate_examples(gallery_ref=gallery_ref, code_below=True)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
examples = sorted(examples, key=lambda x: x["title"])
examples_toc = collections.OrderedDict(
{
"Simple Charts": [],
"Bar Charts": [],
"Line Charts": [],
"Area Charts": [],
"Circular Plots": [],
"Scatter Plots": [],
"Histograms": [],
"Maps": [],
"Interactive Charts": [],
"Case Studies": [],
"Other Charts": [],
}
)
for d in examples:
examples_toc[d["category"]].append(d)
# Write the gallery index file
with open(os.path.join(target_dir, "index.rst"), "w") as f:
f.write(
GALLERY_TEMPLATE.render(
title=gallery_title,
examples=examples_toc.items(),
image_dir="/_static",
gallery_ref=gallery_ref,
)
)
# save the images to file
save_example_pngs(examples, image_dir)
# Write the individual example files
for prev_ex, example, next_ex in prev_this_next(examples):
if prev_ex:
example["prev_ref"] = "gallery_{name}".format(**prev_ex)
if next_ex:
example["next_ref"] = "gallery_{name}".format(**next_ex)
target_filename = os.path.join(target_dir, example["name"] + ".rst")
with open(os.path.join(target_filename), "w", encoding="utf-8") as f:
f.write(EXAMPLE_TEMPLATE.render(example))
def setup(app):
app.connect("builder-inited", main)
app.add_css_file("altair-gallery.css")
app.add_config_value("altair_gallery_dir", "gallery", "env")
app.add_config_value("altair_gallery_ref", "example-gallery", "env")
app.add_config_value("altair_gallery_title", "Example Gallery", "env")
app.add_directive_to_domain("py", "altair-minigallery", AltairMiniGalleryDirective)

View File

@@ -0,0 +1,341 @@
"""
Altair Plot Sphinx Extension
============================
This extension provides a means of inserting live-rendered Altair plots within
sphinx documentation. There are two directives defined: ``altair-setup`` and
``altair-plot``. ``altair-setup`` code is used to set-up various options
prior to running the plot code. For example::
.. altair-plot::
:output: none
from altair import *
import pandas as pd
data = pd.DataFrame({'a': list('CCCDDDEEE'),
'b': [2, 7, 4, 1, 2, 6, 8, 4, 7]})
.. altair-plot::
Chart(data).mark_point().encode(
x='a',
y='b'
)
In the case of the ``altair-plot`` code, the *last statement* of the code-block
should contain the chart object you wish to be rendered.
Options
-------
The directives have the following options::
.. altair-plot::
:namespace: # specify a plotting namespace that is persistent within the doc
:hide-code: # if set, then hide the code and only show the plot
:code-below: # if set, then code is below rather than above the figure
:output: [plot|repr|stdout|none]
:alt: text # Alternate text when plot cannot be rendered
:links: editor source export # specify one or more of these options
:chart-var-name: chart # name of variable in namespace containing output
Additionally, this extension introduces a global configuration
``altairplot_links``, set in your ``conf.py`` which is a dictionary
of links that will appear below plots, unless the ``:links:`` option
again overrides it. It should look something like this::
# conf.py
# ...
altairplot_links = {'editor': True, 'source': True, 'export': True}
# ...
If this configuration is not specified, all are set to True.
"""
import contextlib
import io
import os
import json
import warnings
import jinja2
from docutils import nodes
from docutils.parsers.rst import Directive
from docutils.parsers.rst.directives import flag, unchanged
from sphinx.locale import _
import altair as alt
from altair.utils.execeval import eval_block
# These default URLs can be changed in conf.py; see setup() below.
VEGA_JS_URL_DEFAULT = "https://cdn.jsdelivr.net/npm/vega@{}".format(alt.VEGA_VERSION)
VEGALITE_JS_URL_DEFAULT = "https://cdn.jsdelivr.net/npm/vega-lite@{}".format(
alt.VEGALITE_VERSION
)
VEGAEMBED_JS_URL_DEFAULT = "https://cdn.jsdelivr.net/npm/vega-embed@{}".format(
alt.VEGAEMBED_VERSION
)
VGL_TEMPLATE = jinja2.Template(
"""
<div id="{{ div_id }}">
<script>
// embed when document is loaded, to ensure vega library is available
// this works on all modern browsers, except IE8 and older
document.addEventListener("DOMContentLoaded", function(event) {
var spec = {{ spec }};
var opt = {
"mode": "{{ mode }}",
"renderer": "{{ renderer }}",
"actions": {{ actions}}
};
vegaEmbed('#{{ div_id }}', spec, opt).catch(console.err);
});
</script>
</div>
"""
)
class altair_plot(nodes.General, nodes.Element):
pass
def purge_altair_namespaces(app, env, docname):
if not hasattr(env, "_altair_namespaces"):
return
env._altair_namespaces.pop(docname, {})
DEFAULT_ALTAIRPLOT_LINKS = {"editor": True, "source": True, "export": True}
def validate_links(links):
if links.strip().lower() == "none":
return False
links = links.strip().split()
diff = set(links) - set(DEFAULT_ALTAIRPLOT_LINKS.keys())
if diff:
raise ValueError("Following links are invalid: {}".format(list(diff)))
return {link: link in links for link in DEFAULT_ALTAIRPLOT_LINKS}
def validate_output(output):
output = output.strip().lower()
if output not in ["plot", "repr", "stdout", "none"]:
raise ValueError(":output: flag must be one of [plot|repr|stdout|none]")
return output
class AltairPlotDirective(Directive):
has_content = True
option_spec = {
"hide-code": flag,
"code-below": flag,
"namespace": unchanged,
"output": validate_output,
"alt": unchanged,
"links": validate_links,
"chart-var-name": unchanged,
}
def run(self):
env = self.state.document.settings.env
app = env.app
show_code = "hide-code" not in self.options
code_below = "code-below" in self.options
if not hasattr(env, "_altair_namespaces"):
env._altair_namespaces = {}
namespace_id = self.options.get("namespace", "default")
namespace = env._altair_namespaces.setdefault(env.docname, {}).setdefault(
namespace_id, {}
)
code = "\n".join(self.content)
if show_code:
source_literal = nodes.literal_block(code, code)
source_literal["language"] = "python"
# get the name of the source file we are currently processing
rst_source = self.state_machine.document["source"]
rst_dir = os.path.dirname(rst_source)
rst_filename = os.path.basename(rst_source)
# use the source file name to construct a friendly target_id
serialno = env.new_serialno("altair-plot")
rst_base = rst_filename.replace(".", "-")
div_id = "{}-altair-plot-{}".format(rst_base, serialno)
target_id = "{}-altair-source-{}".format(rst_base, serialno)
target_node = nodes.target("", "", ids=[target_id])
# create the node in which the plot will appear;
# this will be processed by html_visit_altair_plot
plot_node = altair_plot()
plot_node["target_id"] = target_id
plot_node["div_id"] = div_id
plot_node["code"] = code
plot_node["namespace"] = namespace
plot_node["relpath"] = os.path.relpath(rst_dir, env.srcdir)
plot_node["rst_source"] = rst_source
plot_node["rst_lineno"] = self.lineno
plot_node["links"] = self.options.get(
"links", app.builder.config.altairplot_links
)
plot_node["output"] = self.options.get("output", "plot")
plot_node["chart-var-name"] = self.options.get("chart-var-name", None)
if "alt" in self.options:
plot_node["alt"] = self.options["alt"]
result = [target_node]
if code_below:
result += [plot_node]
if show_code:
result += [source_literal]
if not code_below:
result += [plot_node]
return result
def html_visit_altair_plot(self, node):
# Execute the code, saving output and namespace
namespace = node["namespace"]
try:
f = io.StringIO()
with contextlib.redirect_stdout(f):
chart = eval_block(node["code"], namespace)
stdout = f.getvalue()
except Exception as e:
warnings.warn(
"altair-plot: {}:{} Code Execution failed:"
"{}: {}".format(
node["rst_source"], node["rst_lineno"], e.__class__.__name__, str(e)
)
)
raise nodes.SkipNode
chart_name = node["chart-var-name"]
if chart_name is not None:
if chart_name not in namespace:
raise ValueError(
"chart-var-name='{}' not present in namespace" "".format(chart_name)
)
chart = namespace[chart_name]
output = node["output"]
if output == "none":
raise nodes.SkipNode
elif output == "stdout":
if not stdout:
raise nodes.SkipNode
else:
output_literal = nodes.literal_block(stdout, stdout)
output_literal["language"] = "none"
node.extend([output_literal])
elif output == "repr":
if chart is None:
raise nodes.SkipNode
else:
rep = " " + repr(chart).replace("\n", "\n ")
repr_literal = nodes.literal_block(rep, rep)
repr_literal["language"] = "none"
node.extend([repr_literal])
elif output == "plot":
if isinstance(chart, alt.TopLevelMixin):
# Last line should be a chart; convert to spec dict
try:
spec = chart.to_dict()
except alt.utils.schemapi.SchemaValidationError:
raise ValueError("Invalid chart: {0}".format(node["code"]))
actions = node["links"]
# TODO: add an option to save spects to file & load from there.
# TODO: add renderer option
# Write spec to a *.vl.json file
# dest_dir = os.path.join(self.builder.outdir, node['relpath'])
# if not os.path.exists(dest_dir):
# os.makedirs(dest_dir)
# filename = "{0}.vl.json".format(node['target_id'])
# dest_path = os.path.join(dest_dir, filename)
# with open(dest_path, 'w') as f:
# json.dump(spec, f)
# Pass relevant info into the template and append to the output
html = VGL_TEMPLATE.render(
div_id=node["div_id"],
spec=json.dumps(spec),
mode="vega-lite",
renderer="canvas",
actions=json.dumps(actions),
)
self.body.append(html)
else:
warnings.warn(
"altair-plot: {}:{} Malformed block. Last line of "
"code block should define a valid altair Chart object."
"".format(node["rst_source"], node["rst_lineno"])
)
raise nodes.SkipNode
def generic_visit_altair_plot(self, node):
# TODO: generate PNGs and insert them here
if "alt" in node.attributes:
self.body.append(_("[ graph: %s ]") % node["alt"])
else:
self.body.append(_("[ graph ]"))
raise nodes.SkipNode
def depart_altair_plot(self, node):
return
def builder_inited(app):
app.add_js_file(app.config.altairplot_vega_js_url)
app.add_js_file(app.config.altairplot_vegalite_js_url)
app.add_js_file(app.config.altairplot_vegaembed_js_url)
def setup(app):
setup.app = app
setup.config = app.config
setup.confdir = app.confdir
app.add_config_value("altairplot_links", DEFAULT_ALTAIRPLOT_LINKS, "env")
app.add_config_value("altairplot_vega_js_url", VEGA_JS_URL_DEFAULT, "html")
app.add_config_value("altairplot_vegalite_js_url", VEGALITE_JS_URL_DEFAULT, "html")
app.add_config_value(
"altairplot_vegaembed_js_url", VEGAEMBED_JS_URL_DEFAULT, "html"
)
app.add_directive("altair-plot", AltairPlotDirective)
app.add_css_file("altair-plot.css")
app.add_node(
altair_plot,
html=(html_visit_altair_plot, depart_altair_plot),
latex=(generic_visit_altair_plot, depart_altair_plot),
texinfo=(generic_visit_altair_plot, depart_altair_plot),
text=(generic_visit_altair_plot, depart_altair_plot),
man=(generic_visit_altair_plot, depart_altair_plot),
)
app.connect("env-purge-doc", purge_altair_namespaces)
app.connect("builder-inited", builder_inited)
return {"version": "0.1"}

View File

@@ -0,0 +1,195 @@
import importlib
import warnings
import re
from docutils.parsers.rst import Directive
from docutils import nodes, utils
from sphinx import addnodes
from recommonmark.parser import CommonMarkParser
def type_description(schema):
"""Return a concise type description for the given schema"""
if not schema or not isinstance(schema, dict) or schema.keys() == {"description"}:
return "any"
elif "$ref" in schema:
return ":class:`{}`".format(schema["$ref"].split("/")[-1])
elif "enum" in schema:
return "[{}]".format(", ".join(repr(s) for s in schema["enum"]))
elif "type" in schema:
if isinstance(schema["type"], list):
return "[{}]".format(", ".join(schema["type"]))
elif schema["type"] == "array":
return "array({})".format(type_description(schema.get("items", {})))
elif schema["type"] == "object":
return "dict"
else:
return "`{}`".format(schema["type"])
elif "anyOf" in schema:
return "anyOf({})".format(
", ".join(type_description(s) for s in schema["anyOf"])
)
else:
warnings.warn(
"cannot infer type for schema with keys {}" "".format(schema.keys())
)
return "--"
def prepare_table_header(titles, widths):
"""Build docutil empty table"""
ncols = len(titles)
assert len(widths) == ncols
tgroup = nodes.tgroup(cols=ncols)
for width in widths:
tgroup += nodes.colspec(colwidth=width)
header = nodes.row()
for title in titles:
header += nodes.entry("", nodes.paragraph(text=title))
tgroup += nodes.thead("", header)
tbody = nodes.tbody()
tgroup += tbody
return nodes.table("", tgroup), tbody
reClassDef = re.compile(r":class:`([^`]+)`")
reCode = re.compile(r"`([^`]+)`")
def add_class_def(node, classDef):
"""Add reference on classDef to node"""
ref = addnodes.pending_xref(
reftarget=classDef,
reftype="class",
refdomain="py", # py:class="None" py:module="altair" refdoc="user_guide/marks"
refexplicit=False,
# refdoc="",
refwarn=False,
)
ref["py:class"] = "None"
ref["py:module"] = "altair"
ref += nodes.literal(text=classDef, classes=["xref", "py", "py-class"])
node += ref
return node
def add_text(node, text):
"""Add text with inline code to node"""
is_text = True
for part in reCode.split(text):
if part:
if is_text:
node += nodes.Text(part, part)
else:
node += nodes.literal(part, part)
is_text = not is_text
return node
def build_row(item):
"""Return nodes.row with property description"""
prop, propschema, required = item
row = nodes.row()
# Property
row += nodes.entry("", nodes.paragraph(text=prop), classes=["vl-prop"])
# Type
str_type = type_description(propschema)
par_type = nodes.paragraph()
is_text = True
for part in reClassDef.split(str_type):
if part:
if is_text:
add_text(par_type, part)
else:
add_class_def(par_type, part)
is_text = not is_text
# row += nodes.entry('')
row += nodes.entry("", par_type) # , classes=["vl-type-def"]
# Description
md_parser = CommonMarkParser()
# str_descr = "***Required.*** " if required else ""
str_descr = ""
str_descr += propschema.get("description", " ")
doc_descr = utils.new_document("schema_description")
md_parser.parse(str_descr, doc_descr)
# row += nodes.entry('', *doc_descr.children, classes="vl-decsr")
row += nodes.entry("", *doc_descr.children, classes=["vl-decsr"])
return row
def build_schema_tabel(items):
"""Return schema table of items (iterator of prop, schema.item, requred)"""
table, tbody = prepare_table_header(
["Property", "Type", "Description"], [10, 20, 50]
)
for item in items:
tbody += build_row(item)
return table
def select_items_from_schema(schema, props=None):
"""Return iterator (prop, schema.item, requred) on prop, return all in None"""
properties = schema.get("properties", {})
required = schema.get("required", [])
if not props:
for prop, item in properties.items():
yield prop, item, prop in required
else:
for prop in props:
try:
yield prop, properties[prop], prop in required
except KeyError:
warnings.warn("Can't find property:", prop)
def prepare_schema_tabel(schema, props=None):
items = select_items_from_schema(schema, props)
return build_schema_tabel(items)
class AltairObjectTableDirective(Directive):
"""
Directive for building a table of attribute descriptions.
Usage:
.. altair-object-table:: altair.MarkConfig
"""
has_content = False
required_arguments = 1
def run(self):
objectname = self.arguments[0]
modname, classname = objectname.rsplit(".", 1)
module = importlib.import_module(modname)
cls = getattr(module, classname)
schema = cls.resolve_references(cls._schema)
# create the table from the object
table = prepare_schema_tabel(schema)
return [table]
def setup(app):
app.add_directive("altair-object-table", AltairObjectTableDirective)

View File

@@ -0,0 +1,199 @@
import ast
import hashlib
import itertools
import json
import re
def create_thumbnail(image_filename, thumb_filename, window_size=(280, 160)):
"""Create a thumbnail whose shortest dimension matches the window"""
from PIL import Image
im = Image.open(image_filename)
im_width, im_height = im.size
width, height = window_size
width_factor, height_factor = width / im_width, height / im_height
if width_factor > height_factor:
final_width = width
final_height = int(im_height * width_factor)
else:
final_height = height
final_width = int(im_width * height_factor)
thumb = im.resize((final_width, final_height), Image.ANTIALIAS)
thumb.save(thumb_filename)
def create_generic_image(filename, shape=(200, 300), gradient=True):
"""Create a generic image"""
from PIL import Image
import numpy as np
assert len(shape) == 2
arr = np.zeros((shape[0], shape[1], 3))
if gradient:
# gradient from gray to white
arr += np.linspace(128, 255, shape[1])[:, None]
im = Image.fromarray(arr.astype("uint8"))
im.save(filename)
SYNTAX_ERROR_DOCSTRING = """
SyntaxError
===========
Example script with invalid Python syntax
"""
def _parse_source_file(filename):
"""Parse source file into AST node
Parameters
----------
filename : str
File path
Returns
-------
node : AST node
content : utf-8 encoded string
Notes
-----
This function adapted from the sphinx-gallery project; license: BSD-3
https://github.com/sphinx-gallery/sphinx-gallery/
"""
with open(filename, "r", encoding="utf-8") as fid:
content = fid.read()
# change from Windows format to UNIX for uniformity
content = content.replace("\r\n", "\n")
try:
node = ast.parse(content)
except SyntaxError:
node = None
return node, content
def get_docstring_and_rest(filename):
"""Separate ``filename`` content between docstring and the rest
Strongly inspired from ast.get_docstring.
Parameters
----------
filename: str
The path to the file containing the code to be read
Returns
-------
docstring: str
docstring of ``filename``
category: list
list of categories specified by the "# category:" comment
rest: str
``filename`` content without the docstring
lineno: int
the line number on which the code starts
Notes
-----
This function adapted from the sphinx-gallery project; license: BSD-3
https://github.com/sphinx-gallery/sphinx-gallery/
"""
node, content = _parse_source_file(filename)
# Find the category comment
find_category = re.compile(r"^#\s*category:\s*(.*)$", re.MULTILINE)
match = find_category.search(content)
if match is not None:
category = match.groups()[0]
# remove this comment from the content
content = find_category.sub("", content)
else:
category = None
if node is None:
return SYNTAX_ERROR_DOCSTRING, category, content, 1
if not isinstance(node, ast.Module):
raise TypeError(
"This function only supports modules. "
"You provided {}".format(node.__class__.__name__)
)
try:
# In python 3.7 module knows its docstring.
# Everything else will raise an attribute error
docstring = node.docstring
import tokenize
from io import BytesIO
ts = tokenize.tokenize(BytesIO(content).readline)
ds_lines = 0
# find the first string according to the tokenizer and get
# it's end row
for tk in ts:
if tk.exact_type == 3:
ds_lines, _ = tk.end
break
# grab the rest of the file
rest = "\n".join(content.split("\n")[ds_lines:])
lineno = ds_lines + 1
except AttributeError:
# this block can be removed when python 3.6 support is dropped
if (
node.body
and isinstance(node.body[0], ast.Expr)
and isinstance(node.body[0].value, (ast.Str, ast.Constant))
):
docstring_node = node.body[0]
docstring = docstring_node.value.s
# python2.7: Code was read in bytes needs decoding to utf-8
# unless future unicode_literals is imported in source which
# make ast output unicode strings
if hasattr(docstring, "decode") and not isinstance(docstring, str):
docstring = docstring.decode("utf-8")
# python3.8: has end_lineno
lineno = (
getattr(docstring_node, "end_lineno", None) or docstring_node.lineno
) # The last line of the string.
# This get the content of the file after the docstring last line
# Note: 'maxsplit' argument is not a keyword argument in python2
rest = content.split("\n", lineno)[-1]
lineno += 1
else:
docstring, rest = "", ""
if not docstring:
raise ValueError(
(
'Could not find docstring in file "{0}". '
"A docstring is required for the example gallery."
).format(filename)
)
return docstring, category, rest, lineno
def prev_this_next(it, sentinel=None):
"""Utility to return (prev, this, next) tuples from an iterator"""
i1, i2, i3 = itertools.tee(it, 3)
next(i3, None)
return zip(itertools.chain([sentinel], i1), i2, itertools.chain(i3, [sentinel]))
def dict_hash(dct):
"""Return a hash of the contents of a dictionary"""
serialized = json.dumps(dct, sort_keys=True)
try:
m = hashlib.md5(serialized)
except TypeError:
m = hashlib.md5(serialized.encode())
return m.hexdigest()