docs, tools: add local link checker

This commit adds tools/check_links.py script, that checks that all local
links from documentation files are valid; fixes some of the invalid links
that we had; and makes travis buildbot check them as well.

Makefile

@@ -132,7 +132,7 @@ endif
 	execprog mutate prog2c stress repro upgrade db \
 	bin/syz-sysgen bin/syz-extract bin/syz-fmt \
 	extract generate \
-	format tidy test arch presubmit clean
+	format tidy test check_links arch presubmit clean
 
 all: host target
 
@@ -247,6 +247,7 @@ arch:
 	env TARGETOS=netbsd TARGETARCH=amd64 $(MAKE) target
 
 presubmit:
+	$(MAKE) check_links
 	$(MAKE) generate
 	$(MAKE) all
 	$(MAKE) arch
@@ -259,3 +260,6 @@ clean:
 # For a tupical Ubuntu/Debian distribution, requires sudo.
 install_prerequisites:
 	apt-get install libc6-dev-i386 lib32stdc++-4.8-dev linux-libc-dev g++-aarch64-linux-gnu g++-powerpc64le-linux-gnu g++-arm-linux-gnueabihf
+
+check_links:
+	python ./tools/check_links.py $$(pwd) $$(ls ./*.md; find ./docs/ -name '*.md')

docs/found_bugs_usb.md

@@ -45,7 +45,7 @@
 * [usb/net/hso: warning in hso_free_net_device](https://groups.google.com/forum/#!topic/syzkaller/J4Ua_T43Tjw)
 * [usb/net/hso: global-out-of-bounds in hso_probe](https://groups.google.com/forum/#!topic/syzkaller/TDPcSXI2nBA)
 * [usb/media/smsusb: use-after-free in worker_thread](https://groups.google.com/forum/#!topic/syzkaller/RS7QUTKo23s)
-* [usb/storage/uas: slab-out-of-bounds in uas_probe](https://groups.google.com/forum/#!topic/syzkaller/pCswO77gRlM) [[fix](786de92b3cb26012d3d0f00ee37adf14527f35c4)]
+* [usb/storage/uas: slab-out-of-bounds in uas_probe](https://groups.google.com/forum/#!topic/syzkaller/pCswO77gRlM) [[fix](https://github.com/torvalds/linux/commit/786de92b3cb26012d3d0f00ee37adf14527f35c4)]
 * [usb/sound/usx2y: warning in usb_stream_new/__alloc_pages_slowpath](https://groups.google.com/forum/#!topic/syzkaller/vGwGJW_H-0I) [[fix](https://github.com/torvalds/linux/commit/7682e399485fe19622b6fd82510b1f4551e48a25)]
 * [usb/media/pvrusb2: warning in pvr2_send_request_ex/usb_submit_urb](https://groups.google.com/forum/#!topic/syzkaller/M2GeMYS0f6E)
 * [usb/media/smsusb: null-ptr-deref in smsusb_init_device](https://groups.google.com/forum/#!topic/syzkaller/yvhFawNoqxE)

docs/internals.md

@@ -45,7 +45,7 @@ and up to 100 `logN` and `reportN` files, one pair per test machine crash:
      ...
 ```
 
-Descriptions are extracted using a set of [regular expressions](/report/report.go#L33).
+Descriptions are extracted using a set of [regular expressions](/pkg/report/report.go#L33).
 This set may need to be extended if you are using a different kernel architecture, or are just seeing a previously unseen kernel error messages.
 
 `logN` files contain raw `syzkaller` logs and include kernel console output as well as programs executed before the crash.

docs/setup_generic.md

@@ -48,7 +48,7 @@ These are the generic requirements for a syzkaller VM:
    the debugfs filesystem at `/sys/kernel/debug`.
 
 To use QEMU syzkaller VMs you have to install QEMU on your host system, see [QEMU docs](http://wiki.qemu.org/Manual) for details.
-The [create-image.sh](tools/create-image.sh) script can be used to create a suitable Linux image.
+The [create-image.sh](/tools/create-image.sh) script can be used to create a suitable Linux image.
 Detailed steps for setting up syzkaller with QEMU on a Linux host are avaialble for [x86-64](setup_ubuntu-host_qemu-vm_x86-64-kernel.md) and [arm64](setup_linux-host_qemu-vm_arm64-kernel.md) kernels.
 
 For some details on fuzzing the kernel on an Android device check out [this page](setup_linux-host_android-device_arm64-kernel.md) and the explicit instructions for an Odroid C2 board are available [here](setup_ubuntu-host_odroid-c2-board_arm64-kernel.md).

docs/setup_ubuntu-host_qemu-vm_x86-64-kernel.md

@@ -78,7 +78,7 @@ CONFIG_KASAN=y
 CONFIG_KASAN_INLINE=y
 ```
 
-You might also want to enable some other kernel configs as described [here](linux-kernel-configs.md).
+You might also want to enable some other kernel configs as described [here](linux_kernel_configs.md).
 
 Since enabling these options results in more sub options being available, we need to regenerate config. Run this and press enter each time when prompted for some config value to leave it as default:
 ``` bash

tools/check_links.py

@@ -0,0 +1,66 @@
+#!/usr/bin/python
+
+import os
+import re
+import sys
+
+link_re = re.compile('\[' + '[^\[\]]+' + '\]' + '\(' + '([^\(\)]+)' + '\)')
+
+if len(sys.argv) < 3:
+	print 'Usage: <root_dir> <doc_files>...'
+	sys.exit(1)
+
+root = sys.argv[1]
+docs = sys.argv[2:]
+
+links = []
+
+for doc in docs:
+	with open(doc) as f:
+		data = f.read()
+		r = link_re.findall(data)
+		for link in r:
+			links += [(doc, link)]
+
+def filter_link((doc, link)):
+	if link.startswith('http'):
+		return False
+	if link.startswith('#'):
+		return False
+	return True
+
+links = filter(filter_link, links)
+
+def fix_link((doc, link)):
+	link = link.split('#')[0]
+	link = link.split('?')[0]
+	return (doc, link)
+
+links = map(fix_link, links)
+
+errors = []
+
+def check_link((doc, link)):
+	path = os.path.dirname(doc)
+	full_link = None
+	if link[0] == '/':
+		link = link[1:]
+		full_link = os.path.join(root, link)
+	else:
+		full_link = os.path.join(root, path, link)
+	if not os.path.exists(full_link):
+		return False
+	return True
+
+for link in links:
+	if not check_link(link):
+		errors += [link]
+
+if len(errors) == 0:
+	print '%d links checked: OK' % (len(links),)
+	sys.exit(0)
+
+for (doc, link) in errors:
+	print 'File %s linked from %s not found' % (link, doc)
+
+sys.exit(2)