From e5c48394f610f0e6d6d984ee9a07706611f7e649 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Thu, 22 Aug 2024 09:46:32 -0700 Subject: [PATCH 1/4] Fix LT-21872 (Import Phonology crashes) and boundary marker problem --- .../ApplicationServices/XmlImportData.cs | 12 +++++++++- .../DomainImpl/OverridesLing_Lex.cs | 15 +++++++++++++ src/SIL.LCModel/InterfaceAdditions.cs | 6 +++++ .../DomainServices/PhonologyServicesTest.cs | 22 +++++++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs b/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs index 6b265d47..67d9061e 100644 --- a/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs +++ b/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs @@ -995,6 +995,13 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse) cmo = m_cache.LangProject.LexDbOA; Debug.Assert(cmo != null); break; + case "PhBdryMarker": + IPhBdryMarkerFactory factory = m_cache.ServiceLocator.GetInstance(); + Guid guid = Guid.Parse(xrdr.GetAttribute("Guid")); + cmo = factory.Create(guid, (IPhPhonemeSet)fi.Owner); + // Remove the default code added by PhTerminalUnit.SetDefaultValuesAfterInit in OverridesLing_Lex. + (cmo as PhBdryMarker).CodesOS.Clear(); + break; case "PhPhonData": cmo = m_cache.LangProject.PhonologicalDataOA; Debug.Assert(cmo != null); @@ -1003,6 +1010,10 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse) cmo = m_cache.LangProject.PhFeatureSystemOA; Debug.Assert(cmo != null); break; + case "PhPhonemeSet": + cmo = m_cache.LangProject.PhonologicalDataOA.GetPhonemeSet(); + Debug.Assert(cmo != null); + break; default: int clid = m_mdc.GetClassId(sClass); if (fi != null) @@ -1048,7 +1059,6 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse) cmo = m_repoCmObject.GetObject(hvo); // Remove the default code added by PhTerminalUnit.SetDefaultValuesAfterInit in OverridesLing_Lex. (cmo as PhPhoneme)?.CodesOS.Clear(); - (cmo as PhBdryMarker)?.CodesOS.Clear(); // Remove default values. (cmo as PhRegularRule)?.RightHandSidesOS.Clear(); } diff --git a/src/SIL.LCModel/DomainImpl/OverridesLing_Lex.cs b/src/SIL.LCModel/DomainImpl/OverridesLing_Lex.cs index 58959921..834c52fd 100644 --- a/src/SIL.LCModel/DomainImpl/OverridesLing_Lex.cs +++ b/src/SIL.LCModel/DomainImpl/OverridesLing_Lex.cs @@ -6858,6 +6858,21 @@ public List AllPhonemes() return phReps; } + /// + /// Get the PhonemeSet. + /// Hides the fact that multiple PhonemeSets used to be allowed. + /// + /// + /// + public IPhPhonemeSet GetPhonemeSet() + { + if (this.PhonemeSetsOS.Count == 0) + throw new Exception("Missing PhonemeSet."); + if (this.PhonemeSetsOS.Count > 1) + throw new Exception("Too many PhonemeSets."); + return this.PhonemeSetsOS[0]; + } + /// /// Return the list of abbreviations for all the natural classes defined, each in the /// default analysis writing system. diff --git a/src/SIL.LCModel/InterfaceAdditions.cs b/src/SIL.LCModel/InterfaceAdditions.cs index e81c3f19..81d40bc6 100644 --- a/src/SIL.LCModel/InterfaceAdditions.cs +++ b/src/SIL.LCModel/InterfaceAdditions.cs @@ -5626,6 +5626,12 @@ public partial interface IPhPhonData /// List AllNaturalClassAbbrs(); + /// + /// Get the PhonemeSet. + /// Hides the fact that the data structure used to allow more than one. + /// + IPhPhonemeSet GetPhonemeSet(); + /// /// Rebuild the list of PhonRuleFeats /// diff --git a/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs b/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs index 83ac1d96..21fad3f7 100644 --- a/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs +++ b/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs @@ -125,7 +125,14 @@ private void TestXml(string xml, string vernWs) { m_cache.ServiceLocator.WritingSystems.DefaultVernacularWritingSystem = m_cache.ServiceLocator.WritingSystemManager.Get(vernWs); + if (m_cache.LangProject.PhonologicalDataOA.PhonemeSetsOS.Count == 0) + { + var phonemeset = m_cache.ServiceLocator.GetInstance().Create(); + m_cache.LangProject.PhonologicalDataOA.PhonemeSetsOS.Add(phonemeset); + } }); + ILcmOwningSequence phonemeList = m_cache.LangProject.PhonologicalDataOA.PhonemeSetsOS; + IPhPhonemeSet phonemeSet = m_cache.LangProject.PhonologicalDataOA.GetPhonemeSet(); var services = new PhonologyServices(m_cache); using (var rdr = new StringReader(xml)) { @@ -134,6 +141,21 @@ private void TestXml(string xml, string vernWs) var xml2 = xdoc2.ToString(); TestXml(xdoc, xdoc2); } + // Verify that the references to the PhonemeSet didn't change. + Assert.IsTrue(ReferenceEquals(phonemeList, m_cache.LangProject.PhonologicalDataOA.PhonemeSetsOS)); + Assert.IsTrue(ReferenceEquals(phonemeSet, m_cache.LangProject.PhonologicalDataOA.GetPhonemeSet())); + // Verify that the boundary markers exist with the right GUIDs. + bool hasMorphBdry = false; + bool hasWordBdry = false; + foreach (var marker in m_cache.LangProject.PhonologicalDataOA.GetPhonemeSet().BoundaryMarkersOC) + { + if (marker.Guid == LangProjectTags.kguidPhRuleMorphBdry) + hasMorphBdry = true; + if (marker.Guid == LangProjectTags.kguidPhRuleWordBdry) + hasWordBdry = true; + } + Assert.IsTrue(hasMorphBdry); + Assert.IsTrue(hasWordBdry); } private void TestXml(XDocument xdoc, XDocument xdoc2) From 97f7618bf697443883bbee23fcb14672e1da21be Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Fri, 23 Aug 2024 09:48:47 -0700 Subject: [PATCH 2/4] Add DeletePhonology, fix LT-21870 (Import phonology crash) --- .../DomainServices/M3ModelExportServices.cs | 16 +- .../DomainServices/PhonologyServices.cs | 54 +++---- .../TestData/HomographDrops/Test.bak | 151 ++++++++++++++++++ .../DomainServices/PhonologyServicesTest.cs | 36 +++++ 4 files changed, 217 insertions(+), 40 deletions(-) create mode 100644 tests/SIL.LCModel.FixData.Tests/TestData/HomographDrops/Test.bak diff --git a/src/SIL.LCModel/DomainServices/M3ModelExportServices.cs b/src/SIL.LCModel/DomainServices/M3ModelExportServices.cs index 0ce09b8c..67f43d02 100644 --- a/src/SIL.LCModel/DomainServices/M3ModelExportServices.cs +++ b/src/SIL.LCModel/DomainServices/M3ModelExportServices.cs @@ -591,12 +591,16 @@ select ExportItemAsReference(constraint, constraints.IndexOf(constraint), "Featu new XAttribute("Id", rhs.Hvo), new XElement("StrucChange", from structChange in rhs.StrucChangeOS select ExportContext(structChange)), - new XElement("InputPOSes", from pos in rhs.InputPOSesRC - select ExportItemAsReference(pos, "RequiredPOS")), - new XElement("ReqRuleFeats", from rrf in rhs.ReqRuleFeatsRC - select ExportItemAsReference(rrf, "RuleFeat")), - new XElement("ExclRuleFeats", from erf in rhs.ExclRuleFeatsRC - select ExportItemAsReference(erf, "RuleFeat")), + // RuleFeats and POS are not part of the phonology. + phonology ? null + : new XElement("InputPOSes", from pos in rhs.InputPOSesRC + select ExportItemAsReference(pos, "RequiredPOS")), + phonology ? null + : new XElement("ReqRuleFeats", from rrf in rhs.ReqRuleFeatsRC + select ExportItemAsReference(rrf, "RuleFeat")), + phonology ? null + : new XElement("ExclRuleFeats", from erf in rhs.ExclRuleFeatsRC + select ExportItemAsReference(erf, "RuleFeat")), new XElement("LeftContext", ExportContext(rhs.LeftContextOA)), new XElement("RightContext", ExportContext(rhs.RightContextOA))))); break; diff --git a/src/SIL.LCModel/DomainServices/PhonologyServices.cs b/src/SIL.LCModel/DomainServices/PhonologyServices.cs index 6449cafa..a2a7a52b 100644 --- a/src/SIL.LCModel/DomainServices/PhonologyServices.cs +++ b/src/SIL.LCModel/DomainServices/PhonologyServices.cs @@ -11,6 +11,7 @@ using SIL.LCModel.Core.Text; using SIL.LCModel.Infrastructure.Impl; using static Icu.Normalization.Normalizer2; +using SIL.LCModel.Core.KernelInterfaces; namespace SIL.LCModel.DomainServices { @@ -74,42 +75,27 @@ public void ImportPhonologyFromXml(TextReader rdr) // () => AssignVernacularWritingSystemToDefaultPhPhonemes(Cache)); } - private void AssignVernacularWritingSystemToDefaultPhPhonemes(LcmCache cache) + /// + /// Clear PhonologicalData and Phonological Features. + /// Don't clear boundary markers. + /// + public void DeletePhonology() { - // For all PhCodes in the default phoneme set, change the writing system from "en" to icuLocale - if (cache.LanguageProject.PhonologicalDataOA.PhonemeSetsOS.Count == 0) - return; - var phSet = cache.LanguageProject.PhonologicalDataOA.PhonemeSetsOS[0]; - int wsVern = m_wsVernId == null - ? cache.DefaultVernWs - : cache.ServiceLocator.WritingSystemManager.Get(m_wsVernId).Handle; - foreach (var phone in phSet.PhonemesOC) - { - foreach (var code in phone.CodesOS) - { - - if (code.Representation.VernacularDefaultWritingSystem.Length == 0) - code.Representation.VernacularDefaultWritingSystem = - TsStringUtils.MakeString(code.Representation.UserDefaultWritingSystem.Text, wsVern); - } - if (phone.Name.VernacularDefaultWritingSystem.Length == 0) - phone.Name.VernacularDefaultWritingSystem = - TsStringUtils.MakeString(phone.Name.UserDefaultWritingSystem.Text, wsVern); - } - foreach (var mrkr in phSet.BoundaryMarkersOC) + NonUndoableUnitOfWorkHelper.Do(Cache.ServiceLocator.GetInstance(), () => { - foreach (var code in mrkr.CodesOS) - { - if (code.Representation.VernacularDefaultWritingSystem.Length == 0) - code.Representation.VernacularDefaultWritingSystem = - TsStringUtils.MakeString(code.Representation.UserDefaultWritingSystem.Text, wsVern); - } - if (mrkr.Name.VernacularDefaultWritingSystem.Length == 0) - mrkr.Name.VernacularDefaultWritingSystem = - TsStringUtils.MakeString(mrkr.Name.UserDefaultWritingSystem.Text, wsVern); - } + IPhPhonData phonData = Cache.LangProject.PhonologicalDataOA; + // Delete what is covered by ImportPhonology. + phonData.ContextsOS.Clear(); + phonData.EnvironmentsOS.Clear(); + phonData.FeatConstraintsOS.Clear(); + phonData.NaturalClassesOS.Clear(); + phonData.GetPhonemeSet().PhonemesOC.Clear(); + // Don't clear phonData.GetPhonemeSet().BoundaryMarkersOC! + // They have GUIDs known to the code. + phonData.PhonRulesOS.Clear(); + Cache.LanguageProject.PhFeatureSystemOA.TypesOC.Clear(); + Cache.LanguageProject.PhFeatureSystemOA.FeaturesOC.Clear(); + }); } - - } } diff --git a/tests/SIL.LCModel.FixData.Tests/TestData/HomographDrops/Test.bak b/tests/SIL.LCModel.FixData.Tests/TestData/HomographDrops/Test.bak new file mode 100644 index 00000000..8fb1c582 --- /dev/null +++ b/tests/SIL.LCModel.FixData.Tests/TestData/HomographDrops/Test.bak @@ -0,0 +1,151 @@ + + + + ++ + + + + + + + + + + + +
+riɔ +
+ + + + + + + +sfx + + + + + + +A suffix is an affix that is attached to the end of a root or stem. + + + + + + +suffix + + +- + + + + + + + + + + + + + + + + + +
+riɔ +
+ + + + + + + + + + + + + + + + +
+riɔ +
+ + + + + + + +MjeTyp + + + + + + + + + + + +Morpheme Types + + + + + + + + + + + + + + + + + + + +Dictionary + + + + +en ru enh-x-source en-x-ref en-x-geo enh + + +en ru enh-x-source en-x-ref en-x-geo enh + + +enh + + +enh enh-x-source + + + + +enh + + + + + +enh enh-x-source + + + \ No newline at end of file diff --git a/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs b/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs index 21fad3f7..65d82b38 100644 --- a/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs +++ b/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs @@ -90,6 +90,24 @@ private void SetDefaultVernacularWritingSystem(LcmCache cache, CoreWritingSystem m_cache.ServiceLocator.WritingSystems.DefaultVernacularWritingSystem = vernWritingSystem); } + /// + /// Test all projects in a directory. + /// + /// + private void TestProjects(string directory) + { + foreach (string subDirectory in Directory.GetDirectories(directory, "*")) + { + foreach (string project in Directory.GetFiles(subDirectory, "*.fwdata")) + { + Console.WriteLine("Testing " + project); + CreateTestCache(); + TestProject(subDirectory, project); + DestroyTestCache(); + } + } + } + private void TestProject(string projectsDirectory, string dbFileName) { var projectId = new TestProjectId(BackendProviderType.kXML, dbFileName); @@ -98,6 +116,15 @@ private void TestProject(string projectsDirectory, string dbFileName) using (var cache = LcmCache.CreateCacheFromExistingData(projectId, "en", m_ui, m_lcmDirectories, new LcmSettings(), new DummyProgressDlg())) { + // Create PhonemeSet if necessary. + NonUndoableUnitOfWorkHelper.Do(m_cache.ActionHandlerAccessor, () => + { + if (m_cache.LangProject.PhonologicalDataOA.PhonemeSetsOS.Count == 0) + { + var phonemeset = m_cache.ServiceLocator.GetInstance().Create(); + m_cache.LangProject.PhonologicalDataOA.PhonemeSetsOS.Add(phonemeset); + } + }); // Export project as XML. var services = new PhonologyServices(cache); XDocument xdoc = services.ExportPhonologyAsXml(); @@ -109,6 +136,7 @@ private void TestProject(string projectsDirectory, string dbFileName) var vernWs = cache.ServiceLocator.WritingSystemManager.Get(cache.DefaultVernWs); SetDefaultVernacularWritingSystem(m_cache, vernWs); var services2 = new PhonologyServices(m_cache, vernWs.Id); + services2.DeletePhonology(); services2.ImportPhonologyFromXml(rdr); xdoc2 = services2.ExportPhonologyAsXml(); } @@ -134,6 +162,7 @@ private void TestXml(string xml, string vernWs) ILcmOwningSequence phonemeList = m_cache.LangProject.PhonologicalDataOA.PhonemeSetsOS; IPhPhonemeSet phonemeSet = m_cache.LangProject.PhonologicalDataOA.GetPhonemeSet(); var services = new PhonologyServices(m_cache); + services.DeletePhonology(); using (var rdr = new StringReader(xml)) { services.ImportPhonologyFromXml(rdr); @@ -156,6 +185,7 @@ private void TestXml(string xml, string vernWs) } Assert.IsTrue(hasMorphBdry); Assert.IsTrue(hasWordBdry); + Assert.IsTrue(m_cache.LangProject.PhonologicalDataOA.GetPhonemeSet().BoundaryMarkersOC.Count == 2); } private void TestXml(XDocument xdoc, XDocument xdoc2) @@ -1211,5 +1241,11 @@ public void TestPhonologicalFeatures() TestXml(xdoc, xdoc2); } } + + // [Test] + public void TestPCProjects() + { + TestProjects("C:\\Users\\PC\\source\\repos\\FieldWorks\\DistFiles\\Projects"); + } } } From 74343ed48668ef05edd7215ad0957b623a0ad1b2 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Fri, 23 Aug 2024 11:00:12 -0700 Subject: [PATCH 3/4] Remove file added by mistake --- .../TestData/HomographDrops/Test.bak | 151 ------------------ 1 file changed, 151 deletions(-) delete mode 100644 tests/SIL.LCModel.FixData.Tests/TestData/HomographDrops/Test.bak diff --git a/tests/SIL.LCModel.FixData.Tests/TestData/HomographDrops/Test.bak b/tests/SIL.LCModel.FixData.Tests/TestData/HomographDrops/Test.bak deleted file mode 100644 index 8fb1c582..00000000 --- a/tests/SIL.LCModel.FixData.Tests/TestData/HomographDrops/Test.bak +++ /dev/null @@ -1,151 +0,0 @@ - - - - -+ - - - - - - - - - - - -
-riɔ -
- - - - - - - -sfx - - - - - - -A suffix is an affix that is attached to the end of a root or stem. - - - - - - -suffix - - -- - - - - - - - - - - - - - - - - - -
-riɔ -
- - - - - - - - - - - - - - - - -
-riɔ -
- - - - - - - -MjeTyp - - - - - - - - - - - -Morpheme Types - - - - - - - - - - - - - - - - - - - -Dictionary - - - - -en ru enh-x-source en-x-ref en-x-geo enh - - -en ru enh-x-source en-x-ref en-x-geo enh - - -enh - - -enh enh-x-source - - - - -enh - - - - - -enh enh-x-source - - - \ No newline at end of file From fb465e18d52e1b10886c22e6a87502e807dbf991 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Mon, 26 Aug 2024 09:17:39 -0700 Subject: [PATCH 4/4] Fix duplicate GUID problem --- .gitignore | 1 + .../ApplicationServices/XmlImportData.cs | 7 +++++- .../DomainServices/PhonologyServicesTest.cs | 24 ------------------- 3 files changed, 7 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index cd6da820..8facc20b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ artifacts/ lib/downloads/ .idea/ .vs/ +*.bak src/SIL.LCModel.Core/KernelInterfaces/Kernel.cs src/SIL.LCModel/Infrastructure/Impl/Generated*.cs diff --git a/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs b/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs index 67d9061e..93af69c4 100644 --- a/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs +++ b/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs @@ -996,9 +996,14 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse) Debug.Assert(cmo != null); break; case "PhBdryMarker": + IPhBdryMarkerRepository repository = m_cache.ServiceLocator.GetInstance(); IPhBdryMarkerFactory factory = m_cache.ServiceLocator.GetInstance(); Guid guid = Guid.Parse(xrdr.GetAttribute("Guid")); - cmo = factory.Create(guid, (IPhPhonemeSet)fi.Owner); + IPhBdryMarker marker; + if (repository.TryGetObject(guid, out marker)) + cmo = marker; + else + cmo = factory.Create(guid, (IPhPhonemeSet)fi.Owner); // Remove the default code added by PhTerminalUnit.SetDefaultValuesAfterInit in OverridesLing_Lex. (cmo as PhBdryMarker).CodesOS.Clear(); break; diff --git a/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs b/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs index 65d82b38..78376e37 100644 --- a/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs +++ b/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs @@ -90,24 +90,6 @@ private void SetDefaultVernacularWritingSystem(LcmCache cache, CoreWritingSystem m_cache.ServiceLocator.WritingSystems.DefaultVernacularWritingSystem = vernWritingSystem); } - /// - /// Test all projects in a directory. - /// - /// - private void TestProjects(string directory) - { - foreach (string subDirectory in Directory.GetDirectories(directory, "*")) - { - foreach (string project in Directory.GetFiles(subDirectory, "*.fwdata")) - { - Console.WriteLine("Testing " + project); - CreateTestCache(); - TestProject(subDirectory, project); - DestroyTestCache(); - } - } - } - private void TestProject(string projectsDirectory, string dbFileName) { var projectId = new TestProjectId(BackendProviderType.kXML, dbFileName); @@ -1241,11 +1223,5 @@ public void TestPhonologicalFeatures() TestXml(xdoc, xdoc2); } } - - // [Test] - public void TestPCProjects() - { - TestProjects("C:\\Users\\PC\\source\\repos\\FieldWorks\\DistFiles\\Projects"); - } } }