Updates to Sanity Source

#32

Author: pingevt

packages/content/lib/content-sources/content-result.js

@@ -9,17 +9,17 @@ class ContentResultDataFile {
    * @type {*}
    */
   content = '';
-	
+
 	/**
-	 * 
-	 * @param {string} localPath 
-	 * @param {string|JSON|Object|Array} content 
+	 *
+	 * @param {string} localPath
+	 * @param {string|JSON|Object|Array} content
 	 */
 	constructor(localPath, content) {
 		this.localPath = localPath;
 		this.content = content;
 	}
-	
+
 	/**
 	 * Returns the raw content if it's already a string,
 	 * otherwise returns the result of JSON.stringify(this.content).
@@ -48,7 +48,7 @@ class ContentResult {
    * @type {Array<string>}
    */
   mediaUrls = [];
-	
+
 	/**
 	 * @param {Array<ContentResultDataFile>} dataFiles All the data files and their contents that should be saved
 	 * @param {Array<string>} mediaUrls All the media files that should be saved
@@ -57,31 +57,60 @@ class ContentResult {
 		this.dataFiles = dataFiles;
 		this.mediaUrls = mediaUrls;
 	}
-	
+
 	/**
-	 * 
-	 * @param {string} localPath 
-	 * @param {*} content 
+	 *
+	 * @param {string} localPath
+	 * @param {*} content
 	 */
 	addDataFile(localPath, content) {
 		this.dataFiles.push(new ContentResultDataFile(localPath, content));
 	}
-	
+
 	/**
-	 * 
-	 * @param {string} url 
+	 *
+	 * @param {string} url
 	 */
 	addMediaUrl(url) {
 		this.mediaUrls.push(url);
 	}
-	
+
 	/**
-	 * 
-	 * @param {Iterable} files 
+	 *
+	 * @param {Iterable} files
 	 */
 	addMediaUrls(files) {
 		this.mediaUrls.push(...files);
 	}
+
+	/**
+	 * @param {ContentResult) otheContentResult
+	 */
+	combine(otheContentResult) {
+		this.addMediaUrls(otheContentResult.mediaUrls);
+
+		otheContentResult.dataFiles.forEach(item => {
+			this.dataFiles.push(item);
+		});
+	}
+
+	/**
+	 *
+	 * @param {string} id
+	 */
+	collate(id) {
+
+		// Collect all data into 1 object.
+		let collatedData = this.dataFiles.reduce((previousValue, currentValue) => {
+			return [...previousValue, ...currentValue.content];
+		}, []);
+
+		// Remove old datafiles.
+		this.dataFiles = [];
+
+		const fileName = `${id}.json`;
+		this.addDataFile(fileName, collatedData);
+	}
 }
 
-export default ContentResult;
+export default ContentResult;

packages/content/lib/content-sources/sanity-source.js

@@ -30,6 +30,7 @@ export class SanityOptions extends SourceOptions {
     textConverters = [],
     limit = 100,
     maxNumPages = -1,
+    combinePaginatedFiles = false,
     pageNumZeroPad = 0,
     ...rest
   } = {}) {
@@ -96,6 +97,12 @@ export class SanityOptions extends SourceOptions {
      */
     this.maxNumPages = maxNumPages;
 
+    /**
+     * To combine paginated files into 1 file.
+     * @type {boolean}
+     */
+    this.combinePaginatedFiles = combinePaginatedFiles;
+
     /**
      * How many zeros to pad each json filename index with. Default is 0
      * @type {number}
@@ -134,13 +141,15 @@ class SanitySource extends ContentSource {
    * @returns {Promise<ContentResult>}
    */
   async fetchContent() {
-    const result = new ContentResult();
+    // const result = new ContentResult();
+    // console.log(this.config);
 
     let queryPromises = [];
     let customQueryPromises = [];
 
     for (const id of this.config.queries) {
       let query = '*[_type == "' + id + '" ]';
+      const result = new ContentResult();
 
       queryPromises.push(await this._fetchPages(id, query, result, {
         start: 0,
@@ -149,14 +158,18 @@ class SanitySource extends ContentSource {
     }
 
     for (const cqd of this.config.customQueries) {
+      const result = new ContentResult();
       customQueryPromises.push(await this._fetchPages(cqd.id, cqd.query, result, {
         start: 0,
         limit: this.config.limit
       }));
     }
 
     return Promise.all([...queryPromises, ...customQueryPromises]).then((values) => {
-      return result;
+      const masterResult = new ContentResult();
+      values.forEach(el => masterResult.combine(el));
+
+      return masterResult;
     }).catch((error) => {
       this.logger.error(`Sync failed: ${error ? error.message || '' : ''}`);
       return error;
@@ -181,14 +194,19 @@ class SanitySource extends ContentSource {
   ) {
 
     const pageNum = params.start / params.limit;
-    const q = query + '[' + params.start + '..' + (params.start + params.limit) + ']';
+    const q = query + '[' + params.start + '..' + (params.start + params.limit - 1) + ']';
     const p = {};
 
     this.logger.debug(`Fetching page ${pageNum} of ${id}`);
 
     return this.client.fetch(q, p).then((content) => {
 
       if (!content || !content.length) {
+        // If we are combining files, we do that here.
+        if (this.config.combinePaginatedFiles) {
+          result.collate(id);
+        }
+
         // Empty result or no more pages left
         return Promise.resolve(result);
       }