port tocJSON scraper to jsdom

Author: avindra

.editorconfig

@@ -0,0 +1,3 @@
+[*]
+indent_size = 2
+indent_style = space

.gitignore

@@ -1,4 +1,6 @@
 *.html
 node_modules
 
+schema.json
+
 *.pdf

src/deprecated_view_nav.js

@@ -7,9 +7,10 @@ const {JSDOM} = require("jsdom");
  * JSDOM chosen due to availability of standard apis
  * in a browser-free environment.
  */
-const createDOM = (category) => {
+const createDOM = (category, scripts = false) => {
     const fData = readFileSync("./" + category + ".html")
-    const dom = new JSDOM(fData);
+    const options = scripts ? { runScripts: "dangerously" } : undefined;
+    const dom = new JSDOM(fData, options);
     return dom;
 }
 

src/env.js

@@ -0,0 +1,41 @@
+#!/usr/bin/env node
+const {categories} =require('./config');
+
+const {normalize} = require('./src/util');
+const {createDOM} = require('./src/env');
+
+const byTitle = (title) => a => a.title === title;
+
+const onlyObjects = a => /-objects\.html$/.test(a.link);
+const toRepresentation = a => a.link.match(/([A-z]+)-objects\.html$/)[1];
+
+/**
+ * Grab all category defintions for navigation
+ * from JavaScript in the page
+ */
+for (const _category of categories) {
+  const category = normalize(_category);
+  const {window} = createDOM(category, true);
+  const {tocJSON} = window;
+
+  if (!tocJSON) {
+    console.log('failure to parse TOC on page', category);
+    continue;
+  }
+
+  const items = tocJSON
+    .find(byTitle('Reference'))
+    .children[0]
+    .children
+    .find(byTitle('Objects and Attributes'))
+    .children
+    .filter(onlyObjects)
+    .map(toRepresentation);
+
+  console.log('objects are', items);
+  // only need a single version of tocJSON
+  break;
+}
+
+
+