Fix dependencies for docs, using a Sphinx extension to generate a dep…

… file, as suggested by @eli-schwart in #15169 (comment)

Actually, the depfile (sphinx.d) is removed by nija after generating
it as shown by a trace.  I do not know why, but touching an .rts
file has the correct behaviour. So I suspoect it is an optimization.

docs/depfile.py

@@ -0,0 +1,69 @@
+# coding=utf-8
+#
+# QEMU depfile generation extension
+#
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# This work is licensed under the terms of the GNU GPLv2 or later.
+# See the COPYING file in the top-level directory.
+
+"""depfile is a Sphinx extension that writes a dependency file for
+   an external build system"""
+
+import os
+import sphinx
+import sys
+from pathlib import Path
+
+__version__ = '1.0'
+
+def get_infiles(env):
+    for x in env.found_docs:
+        yield str(env.doc2path(x))
+        yield from ((os.path.join(env.srcdir, dep)
+                    for dep in env.dependencies[x]))
+    for mod in sys.modules.values():
+        if hasattr(mod, '__file__'):
+            if mod.__file__:
+                yield mod.__file__
+    # this is perhaps going to include unused files:
+    for static_path in env.config.html_static_path + env.config.templates_path:
+        for path in Path(static_path).rglob('*'):
+            yield str(path)
+
+    # also include kdoc script
+    #yield str(env.config.kerneldoc_bin[1])
+
+
+def write_depfile(app, exception):
+    if exception:
+        return
+
+    env = app.env
+    if not env.config.depfile:
+        return
+
+    # Using a directory as the output file does not work great because
+    # its timestamp does not necessarily change when the contents change.
+    # So create a timestamp file.
+    if env.config.depfile_stamp:
+        with open(env.config.depfile_stamp, 'w') as f:
+            print('depfile.py: Writing ' + env.config.depfile_stamp)
+
+    with open(env.config.depfile, 'w') as f:
+        print('depfile.py: Writing ' + env.config.depfile)
+        print((env.config.depfile_stamp or app.outdir) + ": \\", file=f)
+        print(*get_infiles(env), file=f)
+        for x in get_infiles(env):
+            print(x + ":", file=f)
+
+def setup(app):
+    app.add_config_value('depfile', None, 'env')
+    app.add_config_value('depfile_stamp', None, 'env')
+    app.connect('build-finished', write_depfile)
+
+    return dict(
+        version = __version__,
+        parallel_read_safe = True,
+        parallel_write_safe = True
+    )

pdns/recursordist/docs/conf.py

@@ -18,8 +18,8 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 # import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+import sys
+from pathlib import Path
 import guzzle_sphinx_theme
 import datetime
 
@@ -29,13 +29,15 @@
 #
 # needs_sphinx = '1.0'
 
+sys.path.append(str(Path('.').resolve()))
+
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 #extensions = []
 #extensions = ['redjack.sphinx.lua', 'sphinxcontrib.httpdomain', 'sphinxjsondomain']
 extensions = ['redjack.sphinx.lua', 'sphinxcontrib.httpdomain', 'sphinxjsondomain',
-              'sphinxcontrib.fulltoc', 'changelog']
+              'sphinxcontrib.fulltoc', 'changelog', 'depfile']
 primary_domain = 'lua'
 
 # Add any paths that contain templates here, relative to this directory.
@@ -208,3 +210,6 @@
 
 # A list of files that should not be packed into the epub file.
 epub_exclude_files = ['search.html']
+
+depfile = 'sphinx.deps'
+depfile_stamp = 'sphinx.stamp'

pdns/recursordist/docs/depfile.py

@@ -0,0 +1 @@
+../../../docs/depfile.py

pdns/recursordist/meson.build

@@ -735,8 +735,6 @@ dep_conf_distfile = custom_target(
 )
 
 if python.found()
-  find_rst_files = run_command(['find', docs_dir, '-name', '*.rst'])
-  rst_files = find_rst_files.stdout().strip().split('\n')
 
   html_docs = custom_target(
     'html-docs',
@@ -752,7 +750,8 @@ if python.found()
     ],
     output: 'html-docs',
     console: true,
-    depend_files: rst_files,
+    depfile: 'sphinx.deps',
+    depends: [ metricfiles, recrust ], # for generated .rst files
   )
 
   docs_tarball = custom_target(
@@ -775,7 +774,8 @@ if python.found()
     ],
     output: 'PowerDNS-Recursor.pdf',
     console: true,
-    depend_files: rst_files,
+    depfile: 'sphinx.deps',
+    depends: [ metricfiles, recrust ], # for generated .rst files
   )
 
   run_target(