Rfc/issue 115 build time data access (#269)

* graphql server working

* apollo client connect to apollo server

* connected header example using lit apollo

* todo

* todos

* query and client + server refactor

* schema refactoring

* clean up console logging

* alias all @greenwood/cli/data module imports

* avoid paramater destructuring

* graphql example in the header

* multiple schemas

* internal data sources documentation

* shelf refactor and children query integration

* refactor out ApolloQuery

* ability to intercept client.query calls

* basic semi-working implementation

* remove extra config from server context

* have puppeteer wait for graphql requests before returning content

* fix and add test cases for apollo

* merged resolvers not actually working

* multiple queries support

* everything working

* todos

* TODO tracking

* fix fallback apollo client fetch handling

* full test suite

* cache json test cases

* stablize test due to inconsistent data results ordering

* clean up deps

* todo cleanup

* remove forced client call in SSG mode for client

* represent graph through the schema

* updated data docs

* typos and grammer

* typos and community link fixes

nyc.config.js

@@ -3,6 +3,7 @@ module.exports = {
   all: true,
 
   include: [
+    'packages/cli/src/data/*.js',
     'packages/cli/src/lib/*.js',
     'packages/cli/src/lifecycles/*.js',
     'packages/cli/src/plugins/*.js',
@@ -20,7 +21,7 @@ module.exports = {
   checkCoverage: true,
 
   statements: 80,
-  branches: 65,
+  branches: 70,
   functions: 85,
   lines: 80,
 

packages/cli/package.json

@@ -31,6 +31,10 @@
     "@babel/preset-env": "^7.8.3",
     "@babel/runtime": "^7.8.3",
     "@webcomponents/webcomponentsjs": "^2.3.0",
+    "apollo-cache-inmemory": "^1.6.3",
+    "apollo-client": "^2.6.4",
+    "apollo-link-http": "^1.5.16",
+    "apollo-server": "^2.9.12",
     "babel-loader": "^8.0.5",
     "chalk": "^2.4.2",
     "colors": "^1.3.3",
@@ -40,15 +44,19 @@
     "css-loader": "^2.1.1",
     "css-to-string-loader": "^0.1.3",
     "cssnano": "^4.1.10",
+    "deepmerge": "^4.2.2",
     "file-loader": "^3.0.1",
     "filewatcher-webpack-plugin": "^1.2.0",
     "front-matter": "^3.0.1",
     "fs-extra": "^8.1.0",
     "glob-promise": "^3.4.0",
+    "graphql": "^14.5.8",
+    "graphql-tag": "^2.10.1",
     "html-webpack-plugin": "^3.2.0",
     "lit-element": "^2.0.1",
     "lit-redux-router": "^0.9.3",
     "local-web-server": "^2.6.1",
+    "node-fetch": "^2.6.0",
     "postcss-loader": "^3.0.0",
     "postcss-nested": "^4.1.2",
     "postcss-preset-env": "^6.7.0",
@@ -64,4 +72,4 @@
     "webpack-manifest-plugin": "^2.0.4",
     "webpack-merge": "^4.2.1"
   }
-}
+}

packages/cli/src/config/webpack.config.common.js

@@ -80,6 +80,15 @@ module.exports = ({ config, context }) => {
     .map((plugin) => plugin.provider({ config, context }));
 
   return {
+
+    resolve: {
+      extensions: ['.js', '.json', '.gql', '.graphql'],
+      // TODO - determine alias change (e.g. @greenwood/data) and / or seperate package - #278
+      alias: {
+        '@greenwood/cli/data': path.join(__dirname, '..', './data')
+      }
+    },
+
     entry: {
       index: path.join(context.scratchDir, 'app', 'app.js')
     },
@@ -124,6 +133,10 @@ module.exports = ({ config, context }) => {
       }, {
         test: /\.(ttf|eot|svg|jpe?g|png|gif|otf)(\?v=[0-9]\.[0-9]\.[0-9])?$/,
         loader: 'file-loader'
+      }, {
+        test: /\.(graphql|gql)$/,
+        exclude: /node_modules/,
+        loader: 'graphql-tag/loader'
       }]
     },
 

packages/cli/src/data/cache.js

@@ -0,0 +1,52 @@
+const { ApolloClient } = require('apollo-client');
+const createHttpLink = require('apollo-link-http').createHttpLink;
+const crypto = require('crypto');
+const fetch = require('node-fetch');
+const fs = require('fs-extra');
+const { gql } = require('apollo-server');
+const InMemoryCache = require('apollo-cache-inmemory').InMemoryCache;
+const path = require('path');
+
+/* Extract cache server-side */
+module.exports = async (req, context) => {
+
+  return new Promise(async(resolve, reject) => {
+    try {
+      // TODO avoid having to duplicate / replay calls - #272
+      const client = await new ApolloClient({
+        link: createHttpLink({
+          uri: 'http://localhost:4000?q=internal', /* internal flag to prevent looping cache on request */
+          fetch
+        }),
+        cache: new InMemoryCache()
+      });
+
+      /* Take the same query from request, and repeat the query for our server side cache */
+      const { query, variables } = req.body;
+
+      let { data } = await client.query({
+        query: gql`${query}`,
+        variables
+      });
+
+      if (data) {
+        const cache = JSON.stringify(client.extract());
+        const md5 = crypto.createHash('md5').update(cache).digest('hex');
+
+        /* Get the requests entire (full) route and rootRoute to use as reference for designated cache directory */
+        const { origin, referer } = req.headers;
+        const fullRoute = referer.substring(origin.length, referer.length);
+        const rootRoute = fullRoute.substring(0, fullRoute.substring(1, fullRoute.length).indexOf('/') + 1);
+        const targetDir = path.join(context.publicDir, rootRoute);
+        const targetFile = path.join(targetDir, `${md5}-cache.json`);
+
+        await fs.mkdirs(targetDir, { recursive: true });
+        await fs.writeFile(path.join(targetFile), cache, 'utf8');
+      }
+      resolve();
+    } catch (err) {
+      console.error('create cache error', err);
+      reject(err);
+    }
+  });
+}; 

packages/cli/src/data/client.js

@@ -0,0 +1,32 @@
+import { ApolloClient } from 'apollo-client';
+import { InMemoryCache } from 'apollo-cache-inmemory';
+import { HttpLink } from 'apollo-link-http';
+
+const APOLLO_STATE = window.__APOLLO_STATE__; // eslint-disable-line no-underscore-dangle
+const client = new ApolloClient({
+  cache: new InMemoryCache().restore(APOLLO_STATE),
+  link: new HttpLink({
+    uri: 'http://localhost:4000'
+  })
+});
+const backupQuery = client.query;
+
+client.query = (params) => {
+
+  if (APOLLO_STATE) {
+    // __APOLLO_STATE__ defined, in "SSG" mode...
+    return fetch('./cache.json')
+      .then(response => response.json())
+      .then((response) => {
+        // mock client.query response
+        return {
+          data: new InMemoryCache().restore(response).readQuery(params)
+        };
+      });
+  } else {
+    // __APOLLO_STATE__ NOT defined, in "SPA" mode
+    return backupQuery(params);
+  }
+};
+
+export default client;

packages/cli/src/data/queries/children.gql

@@ -0,0 +1,10 @@
+query($parent: String!) {
+  children(parent: $parent) {
+    id,
+    title,
+    link,
+    filePath,
+    fileName,
+    template
+  }
+}

packages/cli/src/data/queries/graph.gql

@@ -0,0 +1,10 @@
+query {
+  graph {
+    id,
+    title,
+    link,
+    filePath,
+    fileName,
+    template
+  }
+}

packages/cli/src/data/queries/hello.gql

@@ -0,0 +1,3 @@
+query {
+  hello
+}

packages/cli/src/data/queries/navigation.gql

@@ -0,0 +1,6 @@
+query {
+  navigation{
+    label,
+    link
+  }
+}

packages/cli/src/data/schema/graph.js

@@ -0,0 +1,126 @@
+const { gql } = require('apollo-server-express');
+
+const getDeriveMetaFromRoute = (route) => {
+  // TODO hardcoded root / depth - #273
+  const root = route.split('/')[1] || '';
+  const label = root
+    .replace('/', '')
+    .replace('-', ' ')
+    .split(' ')
+    .map((word) => `${word.charAt(0).toUpperCase()}${word.substring(1)}`)
+    .join(' ');
+
+  return {
+    label,
+    root
+  }; 
+};
+
+const getPagesFromGraph = async (root, query, context) => {
+  const pages = [];
+  const { graph } = context;
+
+  graph
+    .forEach((page) => {
+      const { route, mdFile, fileName, template } = page;
+      const id = page.label;
+      const { label } = getDeriveMetaFromRoute(route);
+
+      pages.push({
+        id,
+        filePath: mdFile,
+        fileName,
+        template,
+        title: label,
+        link: route
+      });
+    });
+
+  return pages;
+};
+
+const getNavigationFromGraph = async (root, query, context) => {
+  const navigation = {};
+  const { graph } = context;
+
+  graph
+    .forEach((page) => {
+      const { route } = page;
+      const { root, label } = getDeriveMetaFromRoute(route);
+
+      if (root !== '' && !navigation[root]) {
+        navigation[root] = {
+          label,
+          link: `/${root}/`
+        };
+      }
+    });
+
+  // TODO best format for users, hash map? #271
+  return Object.keys(navigation).map((key) => {
+    return navigation[key];
+  });
+};
+
+const getChildrenFromParentRoute = async (root, query, context) => {
+  const pages = [];
+  const { parent } = query;
+  const { graph } = context;
+
+  graph
+    .forEach((page) => {
+      const { route, mdFile, fileName, template } = page;
+      const root = route.split('/')[1];
+
+      if (root.indexOf(parent) >= 0) {
+        const { label } = getDeriveMetaFromRoute(route);
+        const id = page.label;
+
+        pages.push({
+          id,
+          filePath: mdFile,
+          fileName,
+          template,
+          title: label,
+          link: route
+        });
+      }
+    });
+
+  return pages;
+};
+
+const graphTypeDefs = gql`
+  type Page {
+    id: String,
+    filePath: String,
+    fileName: String,
+    template: String,
+    link: String,
+    title: String
+  }
+
+  type Navigation {
+    label: String,
+    link: String
+  }
+
+  type Query {
+    graph: [Page]
+    navigation: [Navigation]
+    children(parent: String): [Page]
+  }
+`;
+
+const graphResolvers = {
+  Query: {
+    graph: getPagesFromGraph,
+    navigation: getNavigationFromGraph,
+    children: getChildrenFromParentRoute
+  }
+};
+
+module.exports = {
+  graphTypeDefs,
+  graphResolvers
+};

packages/cli/src/data/schema/hello.js

@@ -0,0 +1,18 @@
+const { gql } = require('apollo-server');
+
+const helloTypeDefs = gql`
+  type HelloQuery {
+    hello: String
+  }
+`;
+
+const helloResolvers = {
+  HelloQuery: {
+    hello: () => 'Hello world!'
+  }
+};
+
+module.exports = {
+  helloTypeDefs,
+  helloResolvers
+};

packages/cli/src/data/schema/schema.js

@@ -0,0 +1,17 @@
+// TODO merging resolvers not actually working, resolve as part of #21 or #270
+const { makeExecutableSchema } = require('apollo-server-express');
+const { helloTypeDefs, helloResolvers } = require('./hello');
+const { graphTypeDefs, graphResolvers } = require('./graph');
+
+const schema = makeExecutableSchema({
+  typeDefs: [
+    graphTypeDefs,
+    helloTypeDefs
+  ],
+  resolvers: Object.assign({}, 
+    graphResolvers,
+    helloResolvers
+  )
+});
+
+module.exports = schema;