diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..3cf70a7 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @danielsmith-eu diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index c298b21..287e7e5 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -6,6 +6,7 @@ name: Scala CI - Run Tests on: + workflow_dispatch: push: branches: [ "main" ] pull_request: diff --git a/data-quality-profiler/build.sbt b/data-quality-profiler/build.sbt index 60f11ca..cb2e10e 100644 --- a/data-quality-profiler/build.sbt +++ b/data-quality-profiler/build.sbt @@ -1,7 +1,7 @@ -ThisBuild / organization := "uk.gov.ipt.das" -ThisBuild / version := "1.0.0" +ThisBuild / organization := "io.github.6point6" +ThisBuild / version := "1.1.0" -name := "das-data-profiler" +name := "data-quality-profiler-and-rules-engine" scalaVersion := "2.12.15" @@ -63,6 +63,9 @@ libraryDependencies += "com.itextpdf" % "html2pdf" % "4.0.3" // https://mvnrepository.com/artifact/com.github.spullara.mustache.java/compiler libraryDependencies += "com.github.spullara.mustache.java" % "compiler" % "0.9.10" +//avro support +libraryDependencies += "org.apache.avro" % "avro" % "1.11.1" + Test / parallelExecution := true @@ -81,16 +84,6 @@ addArtifact(artifact in (Compile, assembly), assembly) /***/ -publishTo := Some(s"GitHub Apache Maven Packages" at s"https://maven.pkg.github.com/${System.getenv("GITHUB_REPOSITORY")}") -credentials += Credentials( - "GitHub Package Registry", - "maven.pkg.github.com", - System.getenv("GITHUB_REPOSITORY_OWNER"), - System.getenv("GITHUB_TOKEN") -) -publishMavenStyle := true -isSnapshot := true - // these are all to allow local publishing to overwrite the version publishConfiguration := publishConfiguration.value.withOverwrite(true) publishLocalConfiguration := publishLocalConfiguration.value.withOverwrite(true) diff --git a/data-quality-profiler/publish.sbt b/data-quality-profiler/publish.sbt new file mode 100644 index 0000000..a0352c7 --- /dev/null +++ b/data-quality-profiler/publish.sbt @@ -0,0 +1,35 @@ +ThisBuild / organization := "io.github.6point6" +ThisBuild / organizationName := "6point6" +ThisBuild / organizationHomepage := Some(url("https://6point6.co.uk/")) + +ThisBuild / scmInfo := Some( + ScmInfo( + url("https://github.com/6point6/data-quality-profiler-and-rules-engine"), + "scm:git@github.com:6point6/data-quality-profiler-and-rules-engine.git" + ) +) +ThisBuild / developers := List( + Developer( + id = "danielsmith-eu", + name = "Dr. Daniel Alexander Smith", + email = "dan.smith@6point6.co.uk", + url = url("http://danielsmith.eu") + ) +) + +ThisBuild / description := "Data Quality Profiler and Rules Engine." +ThisBuild / licenses := List( + "MIT" -> new URL("https://opensource.org/license/mit/") +) +ThisBuild / homepage := Some(url("https://github.com/6point6/data-quality-profiler-and-rules-engine")) + +// Remove all additional repository other than Maven Central from POM +ThisBuild / pomIncludeRepository := { _ => false } +ThisBuild / publishTo := { + // For accounts created after Feb 2021: + val nexus = "https://s01.oss.sonatype.org/" + //val nexus = "https://oss.sonatype.org/" + if (isSnapshot.value) Some("snapshots" at nexus + "content/repositories/snapshots") + else Some("releases" at nexus + "service/local/staging/deploy/maven2") +} +ThisBuild / publishMavenStyle := true diff --git a/data-quality-profiler/src/main/scala/uk/gov/ipt/das/dataprofiler/value/RecordValue.scala b/data-quality-profiler/src/main/scala/uk/gov/ipt/das/dataprofiler/value/RecordValue.scala index 722b520..422e97a 100644 --- a/data-quality-profiler/src/main/scala/uk/gov/ipt/das/dataprofiler/value/RecordValue.scala +++ b/data-quality-profiler/src/main/scala/uk/gov/ipt/das/dataprofiler/value/RecordValue.scala @@ -86,6 +86,7 @@ object RecordValue { case v: java.util.LinkedHashMap[String, Any] => ProfilableRecord.fromLinkedHashMap(linkedHashMap = v) case v: java.math.BigDecimal => DoubleValue(v.doubleValue()) // TODO do better, write tests against this case v: java.util.ArrayList[_] => ArrayValue(v.asScala.map{ RecordValue.fromAny }) + case v: org.apache.avro.util.Utf8 => StringValue(v.toString) case foo => throw new Exception(s"Unknown primtive type of in RecordValue.fromPrimitive: $foo, class: ${foo.getClass}") } diff --git a/data-quality-profiler/src/test/scala/uk/gov/ipt/das/dataprofiler/value/ValuesTests.scala b/data-quality-profiler/src/test/scala/uk/gov/ipt/das/dataprofiler/value/ValuesTests.scala index e1a0646..98e3295 100644 --- a/data-quality-profiler/src/test/scala/uk/gov/ipt/das/dataprofiler/value/ValuesTests.scala +++ b/data-quality-profiler/src/test/scala/uk/gov/ipt/das/dataprofiler/value/ValuesTests.scala @@ -105,6 +105,11 @@ class ValuesTests extends AnyFunSpec { } + it("should return a string for a avro record value") { + val testValue = RecordValue.fromAny( new org.apache.avro.util.Utf8("foo")) + assertResult(testValue.asString)("foo") + } + } }