From f79e2ecebcf57d4e30932f13c8154b02d12928e1 Mon Sep 17 00:00:00 2001 From: Andrew Valencik Date: Sun, 5 Nov 2023 06:41:05 -0500 Subject: [PATCH 1/6] Upgrade to laika 1.0.0 --- build.sbt | 6 ++--- .../protosearch/analysis/DocsDirectory.scala | 8 +++--- .../protosearch/analysis/IngestMarkdown.scala | 9 +++---- .../analysis/PlaintextRenderer.scala | 25 +++++++++---------- .../DocumentationIndexWriterApp.scala | 4 +-- 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/build.sbt b/build.sbt index 28d5a58c..b38ca560 100644 --- a/build.sbt +++ b/build.sbt @@ -38,7 +38,7 @@ val circeV = "0.14.6" val fs2V = "3.9.2" val http4sDomV = "0.2.10" val http4sV = "0.23.23" -val laikaV = "0.19.5" +val laikaV = "1.0.0" val lucilleV = "0.0.1" val munitCatsEffectV = "2.0.0-M3" val munitV = "1.0.0-M10" @@ -90,8 +90,8 @@ lazy val laikaIO = crossProject(JVMPlatform) "co.fs2" %%% "fs2-io" % fs2V, "org.scodec" %%% "scodec-core" % scodecV(scalaVersion.value), "pink.cozydev" %%% "lucille" % lucilleV, - "org.planet42" %%% "laika-core" % laikaV, - "org.planet42" %%% "laika-io" % laikaV, + "org.typelevel" %%% "laika-core" % laikaV, + "org.typelevel" %%% "laika-io" % laikaV, "org.scalameta" %%% "munit" % munitV % Test, "org.typelevel" %%% "munit-cats-effect" % munitCatsEffectV % Test, ), diff --git a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala index 3d480492..e666205f 100644 --- a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala +++ b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala @@ -20,10 +20,10 @@ import cats.data.NonEmptyList import cats.effect.IO import laika.api.MarkupParser import laika.format.Markdown -import laika.io.implicits._ -import laika.markdown.github.GitHubFlavor -import laika.parse.code.SyntaxHighlighting -import laika.parse.markup.DocumentParser.RendererError +import laika.format.Markdown.GitHubFlavor +import laika.config.SyntaxHighlighting +import laika.api.errors.RendererError +import laika.io.syntax._ object DocsDirectory { diff --git a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IngestMarkdown.scala b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IngestMarkdown.scala index 4d65952f..a1061945 100644 --- a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IngestMarkdown.scala +++ b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IngestMarkdown.scala @@ -27,9 +27,9 @@ import laika.ast.RootElement import laika.ast.Section import laika.ast.Text import laika.format.Markdown -import laika.markdown.github.GitHubFlavor -import laika.parse.code.SyntaxHighlighting -import laika.parse.markup.DocumentParser.RendererError +import laika.format.Markdown.GitHubFlavor +import laika.config.SyntaxHighlighting +import laika.api.errors.{RendererError, TransformationError} case class SubDocument(fileName: String, anchor: Option[String], title: String, content: String) @@ -87,10 +87,9 @@ object IngestMarkdown { subDocs.sequence } - def transform(input: String): Either[RendererError, NonEmptyList[SubDocument]] = + def transform(input: String): Either[TransformationError, NonEmptyList[SubDocument]] = parser .parse(input) - .leftMap(e => RendererError(e.message, e.path)) .flatMap(renderSubDocuments) } diff --git a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/PlaintextRenderer.scala b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/PlaintextRenderer.scala index 27aa8fb1..3ebe7e63 100644 --- a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/PlaintextRenderer.scala +++ b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/PlaintextRenderer.scala @@ -17,21 +17,18 @@ package pink.cozydev.protosearch.analysis import laika.ast._ -import laika.render.TextFormatter -import laika.factory.RenderFormat -import laika.factory.RenderContext -import laika.render.Indentation +import laika.api.format.{Formatter, RenderFormat} -object PlaintextRenderer extends ((TextFormatter, Element) => String) { +object PlaintextRenderer extends ((Formatter, Element) => String) { - private case class Content(content: Seq[Element], options: Options = NoOpt) + private case class Content(content: Seq[Element], options: Options = Options.empty) extends Element with ElementContainer[Element] { type Self = Content def withOptions(options: Options): Content = copy(options = options) } - def apply(fmt: TextFormatter, element: Element): String = { + def apply(fmt: Formatter, element: Element): String = { def renderElement(e: Element): String = { val (elements, _) = e.productIterator.partition(_.isInstanceOf[Element]) @@ -56,15 +53,17 @@ object PlaintextRenderer extends ((TextFormatter, Element) => String) { } } -case object Plaintext extends RenderFormat[TextFormatter] { +case object Plaintext extends RenderFormat[Formatter] { val fileSuffix = "txt" - val defaultRenderer: (TextFormatter, Element) => String = PlaintextRenderer + val defaultRenderer: (Formatter, Element) => String = + PlaintextRenderer - val formatterFactory: RenderContext[TextFormatter] => TextFormatter = PlaintextFormatter + val formatterFactory: Formatter.Context[Formatter] => Formatter = + PlaintextFormatter } -object PlaintextFormatter extends (RenderContext[TextFormatter] => TextFormatter) { - def apply(context: RenderContext[TextFormatter]): TextFormatter = - TextFormatter(context.renderChild, context.root, Nil, Indentation.default) +object PlaintextFormatter extends (Formatter.Context[Formatter] => Formatter) { + def apply(context: Formatter.Context[Formatter]): Formatter = + Formatter.defaultFactory(context.withIndentation(Formatter.Indentation.default)) } diff --git a/searchdocs-io/src/main/scala-3/pink/cozydev/protosearch/DocumentationIndexWriterApp.scala b/searchdocs-io/src/main/scala-3/pink/cozydev/protosearch/DocumentationIndexWriterApp.scala index eb9c1a4d..a70994e2 100644 --- a/searchdocs-io/src/main/scala-3/pink/cozydev/protosearch/DocumentationIndexWriterApp.scala +++ b/searchdocs-io/src/main/scala-3/pink/cozydev/protosearch/DocumentationIndexWriterApp.scala @@ -23,7 +23,7 @@ import io.circe.syntax._ import fs2.{Chunk, Stream} import fs2.io.file.{Files, Path} import fs2.text.utf8 -import laika.parse.markup.DocumentParser.RendererError +import laika.api.errors.TransformationError import pink.cozydev.protosearch.analysis.DocsDirectory import pink.cozydev.protosearch.analysis.SubDocument @@ -35,7 +35,7 @@ object DocumentationIndexWriterApp extends IOApp.Simple { val pathHttp4sDocs = pathHttp4s / "docs/docs/" def collectDocs( - subDocs: Either[RendererError, NonEmptyList[SubDocument]] + subDocs: Either[TransformationError, NonEmptyList[SubDocument]] ): List[Doc] = subDocs match { case Left(_) => Nil // swallow errors From 15cb92f9c9f2af9ebda411d123f9f3227b49b50f Mon Sep 17 00:00:00 2001 From: Andrew Valencik Date: Sun, 5 Nov 2023 21:02:29 -0500 Subject: [PATCH 2/6] Prototype BinaryPostProcessor work --- .../protosearch/analysis/DocsDirectory.scala | 29 ++++++++--- .../protosearch/analysis/IndexFormat.scala | 49 +++++++++++++++++++ 2 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala diff --git a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala index e666205f..2c99d227 100644 --- a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala +++ b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala @@ -16,26 +16,43 @@ package pink.cozydev.protosearch.analysis -import cats.data.NonEmptyList import cats.effect.IO import laika.api.MarkupParser import laika.format.Markdown import laika.format.Markdown.GitHubFlavor import laika.config.SyntaxHighlighting -import laika.api.errors.RendererError import laika.io.syntax._ +import laika.api.Renderer +import cats.effect.IOApp +import cats.effect.kernel.Resource +import laika.api.errors.RendererError +import cats.data.NonEmptyList -object DocsDirectory { +object DocsDirectory extends IOApp.Simple { - val mdParser = MarkupParser + val parserBuilder = MarkupParser .of(Markdown) .using(GitHubFlavor, SyntaxHighlighting) - .withConfigValue("laika.validateLinks", false) + + val parser = parserBuilder .parallel[IO] .build + val config = parserBuilder.config + + val plaintextRenderer = Renderer.of(Plaintext).withConfig(config).parallel[IO].build + + val run = Resource + .both(parser, plaintextRenderer) + .use((parser, renderer) => + parser + .fromDirectory("/home/andrew/src/github.com/cozydev/protosearch/docs") + .parse + .flatMap(tree => renderer.from(tree.root).toDirectory("outputDir").render.void) + ) + def dirToDocs(dirPath: String): IO[Seq[Either[RendererError, NonEmptyList[SubDocument]]]] = - mdParser.use(parser => + parser.use(parser => parser.fromDirectory(dirPath).parse.map { tree => tree.root.allDocuments.map(IngestMarkdown.renderSubDocuments) } diff --git a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala new file mode 100644 index 00000000..f6a96937 --- /dev/null +++ b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala @@ -0,0 +1,49 @@ +package pink.cozydev.protosearch.analysis + +import cats.effect.{Async, Resource} +import fs2.Stream +import laika.api.format.{BinaryPostProcessor, Formatter, TwoPhaseRenderFormat, RenderFormat} +import laika.ast.* +import laika.io.model.{BinaryOutput, RenderedTreeRoot} +import laika.api.builder.OperationConfig +import laika.api.config.Config +import laika.theme.Theme +import java.io.OutputStream + +case object IndexFormat extends TwoPhaseRenderFormat[Formatter, BinaryPostProcessor.Builder] { + + override def description: String = "Protosearch Index" + + def interimFormat: RenderFormat[Formatter] = Plaintext + + def prepareTree(tree: DocumentTreeRoot): Either[Throwable, DocumentTreeRoot] = + Right(tree) // no-op + + /** Post processor that produces the final result based on the interim format. + */ + def postProcessor: BinaryPostProcessor.Builder = new BinaryPostProcessor.Builder { + + def build[F[_]: Async](config: Config, theme: Theme[F]): Resource[F, BinaryPostProcessor[F]] = + Resource.pure[F, BinaryPostProcessor[F]](new BinaryPostProcessor[F] { + + def process( + result: RenderedTreeRoot[F], + output: BinaryOutput[F], + config: OperationConfig, + ): F[Unit] = { + val strs: List[String] = result.allDocuments.map(_.content).toList + val ss: Stream[F, Byte] = fs2.Stream.emits[F, String](strs).map(_.toByte) + + val bytes: Stream[F, Byte] = ss + val outputStream: Resource[F, OutputStream] = output.resource + outputStream.use { os => + val fos = Async[F].pure(os) + val pipe = fs2.io.writeOutputStream(fos) + bytes.through(pipe).compile.drain + } + } + }) + + } + +} From 305a1dda789cf3ba234c2af15365edbe557c1510 Mon Sep 17 00:00:00 2001 From: Andrew Valencik Date: Sun, 5 Nov 2023 21:13:36 -0500 Subject: [PATCH 3/6] Add header to IndexFormat --- .../protosearch/analysis/IndexFormat.scala | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala index f6a96937..0315f3ed 100644 --- a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala +++ b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2022 CozyDev + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package pink.cozydev.protosearch.analysis import cats.effect.{Async, Resource} From 443afc5ea84a738e45adfb96cac98c81512d5b34 Mon Sep 17 00:00:00 2001 From: Andrew Valencik Date: Sun, 5 Nov 2023 21:19:08 -0500 Subject: [PATCH 4/6] Fix formatting to please Scala 2.12 --- .../pink/cozydev/protosearch/analysis/DocsDirectory.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala index 2c99d227..b6202821 100644 --- a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala +++ b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/DocsDirectory.scala @@ -44,12 +44,12 @@ object DocsDirectory extends IOApp.Simple { val run = Resource .both(parser, plaintextRenderer) - .use((parser, renderer) => + .use { case (parser, renderer) => parser .fromDirectory("/home/andrew/src/github.com/cozydev/protosearch/docs") .parse .flatMap(tree => renderer.from(tree.root).toDirectory("outputDir").render.void) - ) + } def dirToDocs(dirPath: String): IO[Seq[Either[RendererError, NonEmptyList[SubDocument]]]] = parser.use(parser => From 2cd6c3f953427c90a3e4602acfed24941c8709bf Mon Sep 17 00:00:00 2001 From: Andrew Valencik Date: Wed, 22 Nov 2023 07:57:00 -0500 Subject: [PATCH 5/6] Inline PlaintextFormatter definition --- .../cozydev/protosearch/analysis/PlaintextRenderer.scala | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/PlaintextRenderer.scala b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/PlaintextRenderer.scala index 3ebe7e63..2fdc9c7a 100644 --- a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/PlaintextRenderer.scala +++ b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/PlaintextRenderer.scala @@ -60,10 +60,5 @@ case object Plaintext extends RenderFormat[Formatter] { PlaintextRenderer val formatterFactory: Formatter.Context[Formatter] => Formatter = - PlaintextFormatter -} - -object PlaintextFormatter extends (Formatter.Context[Formatter] => Formatter) { - def apply(context: Formatter.Context[Formatter]): Formatter = - Formatter.defaultFactory(context.withIndentation(Formatter.Indentation.default)) + context => Formatter.defaultFactory(context.withIndentation(Formatter.Indentation.default)) } From c58629f063588ce2ddb7d94d7d34828e225215db Mon Sep 17 00:00:00 2001 From: Andrew Valencik Date: Sat, 2 Dec 2023 08:29:07 -0500 Subject: [PATCH 6/6] Simplify bytes stream in IndexFormat Co-authored-by: Sam Pillsworth --- .../scala/pink/cozydev/protosearch/analysis/IndexFormat.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala index 0315f3ed..79403860 100644 --- a/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala +++ b/laikaIO/src/main/scala/pink/cozydev/protosearch/analysis/IndexFormat.scala @@ -48,9 +48,7 @@ case object IndexFormat extends TwoPhaseRenderFormat[Formatter, BinaryPostProces config: OperationConfig, ): F[Unit] = { val strs: List[String] = result.allDocuments.map(_.content).toList - val ss: Stream[F, Byte] = fs2.Stream.emits[F, String](strs).map(_.toByte) - - val bytes: Stream[F, Byte] = ss + val bytes: Stream[F, Byte] = fs2.Stream.emits[F, String](strs).map(_.toByte) val outputStream: Resource[F, OutputStream] = output.resource outputStream.use { os => val fos = Async[F].pure(os)